= {
Accept: 'application/vnd.github.v3.raw',
};
// Only add the Authorization header if GITHUB_TOKEN exists
if(process.env.GITHUB_TOKEN) {
headers.Authorization = `token ${process.env.GITHUB_TOKEN}`;
}
const response = await fetch(`https://api.github.com/repos/VSCodium/vscodium/contents/docs/${markdownFile}`, {
headers,
next: { revalidate: 3600 }, // Revalidate every hour
});
if(!response.ok) {
const errorMessage
= response.status === 403
? 'GitHub API rate limit exceeded. Consider adding a GITHUB_TOKEN environment variable for higher limits.'
: `GitHub API responded with status: ${response.status}`;
console.warn(errorMessage);
throw new Error(errorMessage);
}
content = await response.text();
await fse.ensureDir(path.dirname(filePath));
await fse.writeFile(filePath, content);
}
catch (error) {
console.error(`Error fetching doc page ${markdownFile}:`, error);
// Return a fallback page
return {
content: 'Documentation Unavailable
We couldn\'t load the documentation. Please try again later or check the GitHub repository.
',
sections: [],
metadata: {
title: 'Documentation Unavailable',
description: 'We couldn\'t load the documentation. Please try again later.',
},
};
}
}
content = content.replaceAll(/https:\/\/github.com\/VSCodium\/vscodium\/blob\/master\/docs\/([\w-]+)\.md/g, '/docs/$1');
// Extract metadata
const title = extractTitle(content);
const description = extractDescription(content);
const { order } = extractMetadata(content);
// Process markdown
const processor = remark()
.use(remarkGfm) // GitHub Flavored Markdown
.use(remarkRemoveTableOfContents) // Remove "Table of Contents" section
.use(remarkTocWithSlugs) // Custom TOC plugin
.use(html, { sanitize: false }); // Convert to HTML
const result = await processor.process(content);
const htmlContent = result.toString();
// Extract sections from the content
const sections = extractSections(content);
return {
content: htmlContent,
sections,
metadata: {
title,
description,
order,
},
};
});
// Function to get all available doc pages
export const getDocumentationPages = cache(async (): Promise => {
const filePath = path.resolve(process.cwd(), './cache/pages.json');
if(fse.existsSync(filePath)) {
const data = fse.readFileSync(filePath, 'utf8');
return JSON.parse(data) as string[];
}
else {
try {
const headers: Record = {
Accept: 'application/vnd.github.v3+json',
};
if(process.env.GITHUB_TOKEN) {
headers.Authorization = `token ${process.env.GITHUB_TOKEN}`;
}
const response = await fetch('https://api.github.com/repos/VSCodium/vscodium/contents/docs', {
headers,
next: { revalidate: 3600 },
});
if(!response.ok) {
throw new Error(`GitHub API responded with status: ${response.status}`);
}
const files = await response.json() as Array<{ name: string }>;
const markdowns = files.filter((file) => file.name.endsWith('.md')).map((file) => file.name);
await fse.ensureDir(path.dirname(filePath));
await fse.writeFile(filePath, JSON.stringify(markdowns, null, 2));
return markdowns;
}
catch (error) {
console.error('Error fetching doc pages:', error);
return [];
}
}
});
// Function to get all doc pages with their sections
export const getDocumentationPagesWithSections = cache(async (): Promise => {
const documentFiles = await getDocumentationPages();
// Get info for all pages including their sections
const pagesInfo: Array = await Promise.all(
documentFiles.map(async (filename) => {
const slug = filename.replace('.md', '');
const page = await getDocumentationPage(filename);
if(page.metadata.order === 0) {
return null;
}
return {
title: page.metadata.title,
slug,
description: page.metadata.description,
order: page.metadata.order ?? 999, // Default to high number for sorting
sections: page.sections,
};
}),
);
// Sort pages by order then title
return pagesInfo.filter((page) => page !== null).sort((a, b) => {
// Special case for index.md - always first
if(a.slug === 'index') {
return -1;
}
if(b.slug === 'index') {
return 1;
}
// Sort by order if specified
if(a.order !== b.order) {
return (a.order ?? 999) - (b.order ?? 999);
}
// Then alphabetically by title
return a.title.localeCompare(b.title);
});
});