From ca39aa9460c4003f0e204459f4da3fdbb6949189 Mon Sep 17 00:00:00 2001 From: Determinant Date: Mon, 9 Sep 2024 13:59:04 -0700 Subject: finish --- app.mjs | 152 ++++++++++++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 120 insertions(+), 32 deletions(-) diff --git a/app.mjs b/app.mjs index 12bd171..666cb2a 100755 --- a/app.mjs +++ b/app.mjs @@ -3,6 +3,8 @@ const csUrl = "https://aeronav.faa.gov/upload_313-d/supplements/"; const tppUrl = "https://aeronav.faa.gov/upload_313-d/terminal/"; const ifrEnrouteUrl = "https://aeronav.faa.gov/enroute/"; +const vfrUrl = "https://aeronav.faa.gov/visual/"; +const output = "."; import { JSDOM } from "jsdom"; import moment from "moment"; @@ -14,13 +16,16 @@ const csRegions = ["SW"]; const tppRegions = ["SW1", "SW2", "SW3", "SW4"]; const ifrEnrouteLow = [2, 3, 4]; const ifrEnrouteHigh = []; +const vfrSectional = ["San Francisco", "Los Angeles"]; +const vfrTerminal = ["San Francisco", "Los Angeles"]; const getChartSupplementDiretory = async () => { const dom = await JSDOM.fromURL(csUrl, {}); let chd = dom.window.document.querySelector("pre").children; const files = {}; - for (let i = 0; i < csRegions.length; i++) { - files[csRegions[i]] = []; + for (let region of csRegions) { + files[region] = []; + files[region].current = { date: moment(0) }; } const pattern = /CS_([A-Z]+)_([0-9]{8}).pdf/; for (let i = 0; i < chd.length; i++) { @@ -33,10 +38,10 @@ const getChartSupplementDiretory = async () => { const url = `${csUrl}${file}`; if (files[region]) { files[region].push({ url, date }); - if (moment() >= date) { + if (moment() >= date && date > files[region].current.date) { files[region].current = { url, - date: date.format("YYYY-MM-DD"), + date, }; } } @@ -50,8 +55,9 @@ const getTerminalProcedurePublication = async () => { const dom = await JSDOM.fromURL(tppUrl, {}); let chd = dom.window.document.querySelector("pre").children; const files = {}; - for (let i = 0; i < tppRegions.length; i++) { - files[tppRegions[i]] = []; + for (let region of tppRegions) { + files[region] = []; + files[region].current = { date: moment(0) }; } const pattern = /([0-9]{4}-[0-9]{2}-[0-9]{2})/; for (let i = 0; i < chd.length; i++) { @@ -72,10 +78,12 @@ const getTerminalProcedurePublication = async () => { const region = m[1]; if (files[region]) { files[region].push({ url, date }); - files[region].current = { - url, - date: date.format("YYYY-MM-DD"), - }; + if (date > files[region].current.date) { + files[region].current = { + url, + date, + }; + } } } } @@ -93,10 +101,14 @@ const getIFREnroute = async () => { let chd = dom.window.document.querySelector("pre").children; const files = {}; for (const i of ifrEnrouteLow) { - files[`L${i.toFixed(0).padStart(2, "0")}`] = []; + const region = `L${i.toFixed(0).padStart(2, "0")}`; + files[region] = []; + files[region].current = { date: moment(0) }; } for (const i of ifrEnrouteHigh) { - files[`H${i.toFixed(0).padStart(2, "0")}`] = []; + const region = `H${i.toFixed(0).padStart(2, "0")}`; + files[region] = []; + files[region].current = { date: moment(0) }; } const pattern = /([0-9]{2}-[0-9]{2}-[0-9]{4})/; @@ -120,11 +132,13 @@ const getIFREnroute = async () => { unzip[`ENR_${region}.tif`] = ".tif"; if (files[region]) { files[region].push({ url, date }); - files[region].current = { - url, - date: date.format("YYYY-MM-DD"), - unzip, - }; + if (date > files[region].current.date) { + files[region].current = { + url, + date, + unzip, + }; + } } } } @@ -137,20 +151,93 @@ const getIFREnroute = async () => { return files; }; +const getVFR = async () => { + const secFiles = {}; + for (const i of vfrSectional) { + const _region = i.replace(" ", "_"); + secFiles[_region] = []; + secFiles[_region].current = { date: moment(0) }; + } + const tacFiles = {}; + for (const i of vfrTerminal) { + const _region = i.replace(" ", "_"); + tacFiles[_region] = []; + tacFiles[_region].current = { date: moment(0) }; + } + const dom = await JSDOM.fromURL(`${vfrUrl}`, {}); + let chd = dom.window.document.querySelector("pre").children; + const pattern = /([0-9]{2}-[0-9]{2}-[0-9]{4})/; + for (let i = 0; i < chd.length; i++) { + if (chd[i] instanceof dom.window.HTMLAnchorElement) { + const folder = chd[i].textContent; + const m = folder.match(pattern); + if (m) { + const date = moment(m[1], "MM-DD-YYYY"); + if (moment() >= date) { + for (const region of vfrSectional) { + const unzip = {}; + const _region = region.replace(" ", "_"); + const url = `${vfrUrl}${m[1]}/sectional-files/${_region}.zip`; + unzip[`${region} SEC.tif`] = ".tif"; + secFiles[_region].push({ url, date }); + if (date > secFiles[_region].current.date) { + secFiles[_region].current = { + url, + date, + unzip, + }; + } + } + for (const region of vfrTerminal) { + const unzip = {}; + const _region = region.replace(" ", "_"); + const url = `${vfrUrl}${m[1]}/tac-files/${_region}_TAC.zip`; + unzip[`${region} TAC.tif`] = ".tif"; + unzip[`${region} FLY.tif`] = "-flyway.tif"; + tacFiles[_region].push({ url, date }); + if (date > tacFiles[_region].current.date) { + tacFiles[_region].current = { + url, + date, + unzip, + }; + } + } + } else { + console.log(`skipped ${folder}: in the future`); + } + } + } + } + return { secFiles, tacFiles }; +}; + +const vfrCharts = await getVFR(); + const res = [ - //{ prefix: "cs", files: await getChartSupplementDiretory() }, - //{ prefix: "tpp", files: await getTerminalProcedurePublication() }, + { prefix: "cs", files: await getChartSupplementDiretory() }, + { prefix: "tpp", files: await getTerminalProcedurePublication() }, { prefix: "ifr-enroute-low", files: await getIFREnroute() }, + { prefix: "vfr-sectional", files: vfrCharts.secFiles }, + { prefix: "vfr-terminal", files: vfrCharts.tacFiles }, ]; -//console.log(res); +const zip_path = `${output}/zips`; +const chart_path = `${output}/charts`; + +try { + await fs.mkdir(zip_path); + await fs.mkdir(chart_path); +} catch (_) {} + for (let r of res) { for (let region of Object.keys(r.files)) { const cur = r.files[region].current; - if (!cur) { + if (!cur.url) { continue; } - const file = `${r.prefix}-${region.toLowerCase()}-${cur.date}.${cur.unzip ? "zip" : "pdf"}`; + const date = cur.date.format("YYYY-MM-DD"); + const file = `${cur.unzip ? zip_path : chart_path}/${r.prefix}-${region.toLowerCase()}-${date}.${cur.unzip ? "zip" : "pdf"}`; const fileExists = (path) => fs.stat(path).then( () => true, @@ -159,19 +246,20 @@ for (let r of res) { if (await fileExists(file)) { console.log(`file "${file}" already exists`); } else { - console.log(`downloading "${file}"`); + console.log(`downloading "${cur.url}"`); await fetch(cur.url) .then((r) => r.arrayBuffer()) .then((bytes) => fs.writeFile(file, new Uint8Array(bytes))); - if (cur.unzip) { - await decompress(file, "./", { - filter: (f) => cur.unzip[f.path] != null, - map: (f) => { - f.path = `${r.prefix}-${region.toLowerCase()}-${cur.date}${cur.unzip[f.path]}`; - return f; - }, - }); - } + } + if (cur.unzip) { + console.log(`extracting "${file}"`); + await decompress(file, "./", { + filter: (f) => cur.unzip[f.path] != null, + map: (f) => { + f.path = `${chart_path}/${r.prefix}-${region.toLowerCase()}-${date}${cur.unzip[f.path]}`; + return f; + }, + }); } } } -- cgit v1.2.3-70-g09d2