| Hash | Commit message | Author | Date | Files | + | - |
|---|
| 1 | commit 5f851e1c5ee08ebc4bcf345ce42e4c2b0adff4ca |
| 2 | Author: Connor Etherington <[email protected]> |
| 3 | Date: Sat Sep 14 03:03:23 2024 +0200 |
| 4 | |
| 5 | Update 14-September-2024___03:03:23 |
| 6 | --- |
| 7 | app.js | 90 ++++++++++++++++++++++++++++++++++-------------------------------- |
| 8 | 1 file changed, 46 insertions(+), 44 deletions(-) |
| 9 | |
| 10 | diff --git a/app.js b/app.js |
| 11 | index 2d238b4..ca207d5 100755 |
| 12 | --- a/app.js |
| 13 | +++ b/app.js |
| 14 | @@ -245,57 +245,59 @@ const saveFiles = (response) => { |
| 15 | |
| 16 | const scraper = new Scraper(url, config); |
| 17 | |
| 18 | -(async () => { |
| 19 | - try { |
| 20 | - if ((!argv.H && !argv.e && !argv.c) || argv.t) console.log(await scraper.pageText()); |
| 21 | - if (argv.l) console.log(await scraper.links()); |
| 22 | - if (argv.H) console.log(await scraper.getHtml()); |
| 23 | - if (argv.S) console.log(await scraper.scrape(argv.S)); |
| 24 | - if (argv.e) console.log(await scraper.evaluate(argv.e)); |
| 25 | - if (argv.c) console.log(await scraper.cookies()); |
| 26 | - if (argv.save) { |
| 27 | - const response = await scraper.getResponse(); |
| 28 | - saveFiles(response); |
| 29 | +if(require.main == module) { |
| 30 | + (async () => { |
| 31 | + try { |
| 32 | + if ((!argv.H && !argv.e && !argv.c) || argv.t) console.log(await scraper.pageText()); |
| 33 | + if (argv.l) console.log(await scraper.links()); |
| 34 | + if (argv.H) console.log(await scraper.getHtml()); |
| 35 | + if (argv.S) console.log(await scraper.scrape(argv.S)); |
| 36 | + if (argv.e) console.log(await scraper.evaluate(argv.e)); |
| 37 | + if (argv.c) console.log(await scraper.cookies()); |
| 38 | + if (argv.save) { |
| 39 | + const response = await scraper.getResponse(); |
| 40 | + saveFiles(response); |
| 41 | + } |
| 42 | + } catch (error) { |
| 43 | + console.error('An error occurred:', error); |
| 44 | } |
| 45 | - } catch (error) { |
| 46 | - console.error('An error occurred:', error); |
| 47 | - } |
| 48 | -})(); |
| 49 | - |
| 50 | -const exportFunctions = async (url, options = {}) => { |
| 51 | - if (Object.keys(options).length === 0) { |
| 52 | - options = { |
| 53 | - cookies: true, |
| 54 | - links: true, |
| 55 | - html: true, |
| 56 | - text: true, |
| 57 | + })(); |
| 58 | +} else { |
| 59 | + const exportFunctions = async (url, options = {}) => { |
| 60 | + if (Object.keys(options).length === 0) { |
| 61 | + options = { |
| 62 | + cookies: true, |
| 63 | + links: true, |
| 64 | + html: true, |
| 65 | + text: true, |
| 66 | + }; |
| 67 | + } |
| 68 | + |
| 69 | + const scrapeOptions = { |
| 70 | + headers: options.headers || {}, |
| 71 | + method: options.post ? 'POST' : 'GET', |
| 72 | }; |
| 73 | - } |
| 74 | |
| 75 | - const scrapeOptions = { |
| 76 | - headers: options.headers || {}, |
| 77 | - method: options.post ? 'POST' : 'GET', |
| 78 | - }; |
| 79 | + const scraper = new Scraper(url, scrapeOptions); |
| 80 | + await scraper.ensureInitialized(); |
| 81 | |
| 82 | - const scraper = new Scraper(url, scrapeOptions); |
| 83 | - await scraper.ensureInitialized(); |
| 84 | + const results = {}; |
| 85 | |
| 86 | - const results = {}; |
| 87 | + if (options.text) results.text = await scraper.pageText(); |
| 88 | + if (options.links) results.links = await scraper.links(); |
| 89 | + if (options.html) results.html = await scraper.getHtml(); |
| 90 | + if (options.selector) results.scraped = await scraper.scrape(options.selector); |
| 91 | + if (options.eval) results.evaluated = await scraper.evaluate(options.eval); |
| 92 | + if (options.cookies) results.cookies = await scraper.cookies(); |
| 93 | |
| 94 | - if (options.text) results.text = await scraper.pageText(); |
| 95 | - if (options.links) results.links = await scraper.links(); |
| 96 | - if (options.html) results.html = await scraper.getHtml(); |
| 97 | - if (options.selector) results.scraped = await scraper.scrape(options.selector); |
| 98 | - if (options.eval) results.evaluated = await scraper.evaluate(options.eval); |
| 99 | - if (options.cookies) results.cookies = await scraper.cookies(); |
| 100 | + if (options.save) { |
| 101 | + const response = await scraper.getResponse(); |
| 102 | + saveFiles(response); |
| 103 | + } |
| 104 | |
| 105 | - if (options.save) { |
| 106 | - const response = await scraper.getResponse(); |
| 107 | - saveFiles(response); |
| 108 | - } |
| 109 | + return results; |
| 110 | + }; |
| 111 | |
| 112 | - return results; |
| 113 | + module.exports = exportFunctions; |
| 114 | }; |
| 115 | |
| 116 | -module.exports = exportFunctions; |
| 117 | - |