dxrknesss/typst-documentation-fetcher
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

remove unused code

Browse Source
This commit is contained in:
dxrknesss
2025-10-05 12:42:16 +03:00
parent e0252a23b5
commit c130c7a1de
1 changed files with 0 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files
index.ts
-67
View File
@@ -14,18 +14,14 @@ const main = async () => {
const parser = new XMLParser(); const parser = new XMLParser();
const result = parser.parse(sitemap); const result = parser.parse(sitemap);
// console.log(result.urlset);
const urlArr: { loc: string }[] = result.urlset.url; const urlArr: { loc: string }[] = result.urlset.url;
const urls: string[] = urlArr const urls: string[] = urlArr
.map(url => url.loc) .map(url => url.loc)
.filter(url => url.includes("/reference/")); .filter(url => url.includes("/reference/"));
const pages = await Promise.all(urls.map(url => fetchPage(url))) const pages = await Promise.all(urls.map(url => fetchPage(url)))
// const pages = await fetchPages(urls);
const cleanPages = await Promise.all(pages.map(page => stripPage(page))) const cleanPages = await Promise.all(pages.map(page => stripPage(page)))
// const cleanPages = await stripPages(pages);
await Promise.all(cleanPages.map(page => savePage(page))); await Promise.all(cleanPages.map(page => savePage(page)));
// await savePages(cleanPages);
}; };
async function fetchPage(url: string): Promise<Page> { async function fetchPage(url: string): Promise<Page> {
@@ -37,28 +33,6 @@ async function fetchPage(url: string): Promise<Page> {
return { url: url, contents: page }; return { url: url, contents: page };
} }
// async function fetchPages(urls: string[]): Promise<Page[]> {
// const promises = urls.map(url => {
// console.log(`downloading ${url}...`);
// return fetch(url);
// });
//
// const responses = (await Promise.allSettled(promises))
// .filter(fulfilledOnly);
// if (responses.length !== urls.length) {
// throw new Error("responses.length !== urls.length");
// }
//
// const pages = (await Promise.allSettled(responses.map(resp => resp.value.text())))
// .filter(fulfilledOnly)
// .map(res => res.value);
// if (pages.length !== urls.length) {
// throw new Error("responses.length !== urls.length");
// }
//
// return pages.map((page, idx) => ({ url: urls[idx], contents: page }));
// }
const unneededElements = ["header", "nav", "footer", "head", "script"]; const unneededElements = ["header", "nav", "footer", "head", "script"];
async function stripPage(page: Page): Promise<Page> { async function stripPage(page: Page): Promise<Page> {
@@ -71,27 +45,6 @@ async function stripPage(page: Page): Promise<Page> {
return page; return page;
} }
// async function stripPages(pages: Page[]): Promise<Page[]> {
// const unneededElements = ["header", "nav", "footer", "head", "script"];
//
// return (await Promise.allSettled(pages.map(async (page) => {
// console.log(`cleaning page ${page.url}`);
// const $ = cheerio.load(page.contents);
// unneededElements.forEach(tag => $(tag).remove());
// page.contents = $.html();
//
// return page;
// })))
// .filter(fulfilledOnly)
// .map(res => res.value);
//
// }
// function fulfilledOnly<T>(res: PromiseSettledResult<T>) {
// return res.status === "fulfilled";
// }
async function savePage(page: Page) { async function savePage(page: Page) {
const refIdx = page.url.indexOf("/reference/"); const refIdx = page.url.indexOf("/reference/");
let fileName = page.url let fileName = page.url
@@ -110,24 +63,4 @@ async function savePage(page: Page) {
console.log(`wrote ${fileName} to disk!`); console.log(`wrote ${fileName} to disk!`);
} }
// async function savePages(pages: Page[]) {
// await Promise.allSettled(pages.map(page => {
// const refIdx = page.url.indexOf("/reference/");
// let fileName = page.url
// .substring(refIdx + "/reference/".length)
// .replaceAll("/", "_");
//
// if (fileName === "") {
// fileName = "index";
// } else {
// fileName = fileName.slice(0, -1);
// }
// fileName = fileName + ".html";
//
// console.log(`writing ${fileName}`)
//
// Bun.write(`typst-docs/${fileName}`, page.contents)
// }))
// }
main(); main();