From 5c246e7061a52121b5f1bf889ebe707b54f9876a Mon Sep 17 00:00:00 2001 From: Dong Nguyen Date: Thu, 6 Aug 2026 14:42:39 +0700 Subject: [PATCH 1/6] v9.0.0-rc0 - Improve fetching logic --- README.md | 99 ++++++++++++++------------------------ SECURITY.md | 2 +- index.d.ts | 30 +++--------- package.json | 16 +++--- src/main.js | 10 ++-- src/main.test.js | 10 ++-- src/utils/retrieve.js | 44 ++--------------- src/utils/retrieve.test.js | 27 +++++------ 8 files changed, 78 insertions(+), 160 deletions(-) diff --git a/README.md b/README.md index 7002095..5215ffc 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ Load and extract article data. Return a Promise object. ```ts extract(String input) extract(String input, Object parserOptions) -extract(String input, Object parserOptions, Object fetchOptions) +extract(String input, Object parserOptions, Function fetcher) ``` Example: @@ -126,102 +126,73 @@ const article = await extract('https://www.cnbc.com/2022/09/21/what-another-majo console.log(article) ``` -##### `fetchOptions` *optional* +##### `fetcher` *optional* -`fetchOptions` is an object that can have the following properties: +A custom fetch function with the signature `(url: string) => Promise`. +Use this to customize HTTP behavior: proxy, headers, TLS, authentication, timeouts, etc. -- `headers`: to set request headers -- `proxy`: another endpoint to forward the request to -- `agent`: a HTTP proxy agent -- `signal`: AbortController signal or AbortSignal timeout to terminate the request +Defaults to `globalThis.fetch`. -For example, you can use this param to set request headers to fetch as below: +**Node.js** (with proxy via undici): ```js import { extract } from '@extractus/article-extractor' +import { fetch, ProxyAgent } from 'undici' -const url = 'https://www.cnbc.com/2022/09/21/what-another-major-rate-hike-by-the-federal-reserve-means-to-you.html' -const article = await extract(url, {}, { - headers: { - 'user-agent': 'Opera/9.60 (Windows NT 6.0; U; en) Presto/2.1.1' - } -}) +const dispatcher = new ProxyAgent('http://proxy.example.com:8080') +const myFetcher = (url) => fetch(url, { dispatcher }) -console.log(article) +const result = await extract('https://www.cnbc.com/2022/09/21/what-another-major-rate-hike-by-the-federal-reserve-means-to-you.html', {}, myFetcher) ``` -You can also specify a proxy endpoint to load remote content, instead of fetching directly. - -For example: +**Bun** (with proxy): ```js import { extract } from '@extractus/article-extractor' -const url = 'https://www.cnbc.com/2022/09/21/what-another-major-rate-hike-by-the-federal-reserve-means-to-you.html' - -await extract(url, {}, { - headers: { - 'user-agent': 'Opera/9.60 (Windows NT 6.0; U; en) Presto/2.1.1' - }, +const myFetcher = (url) => fetch(url, { proxy: { - target: 'https://your-secret-proxy.io/loadXml?url=', - headers: { - 'Proxy-Authorization': 'Bearer YWxhZGRpbjpvcGVuc2VzYW1l...' - }, - } + url: 'http://proxy.example.com:8080', + }, }) -``` - -Passing requests to proxy is useful while running `@extractus/article-extractor` on browser. View [examples/browser-article-parser](examples/browser-article-parser) as reference example. - -For more info about proxy authentication, please refer [HTTP authentication](https://developer.mozilla.org/en-US/docs/Web/HTTP/Authentication) -For a deeper customization, you can consider using [Proxy](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Proxy) to replace `fetch` behaviors with your own handlers. +const result = await extract('https://www.cnbc.com/2022/09/21/what-another-major-rate-hike-by-the-federal-reserve-means-to-you.html', {}, myFetcher) +``` -Another way to work with proxy is use `agent` option instead of `proxy` as below: +**Deno** (with proxy): ```js -import { extract } from '@extractus/article-extractor' - -import { HttpsProxyAgent } from 'https-proxy-agent' - -const proxy = 'http://abc:RaNdoMpasswORd_country-France@proxy.packetstream.io:31113' +import { extract } from 'npm:@extractus/article-extractor' -const url = 'https://www.cnbc.com/2022/09/21/what-another-major-rate-hike-by-the-federal-reserve-means-to-you.html' - -const article = await extract(url, {}, { - agent: new HttpsProxyAgent(proxy), +const client = Deno.createHttpClient({ + proxy: { url: 'http://localhost:8080' }, }) -console.log('Run article-extractor with proxy:', proxy) -console.log(article) -``` +const myFetcher = (url) => fetch(url, { client }) -For more info about [https-proxy-agent](https://www.npmjs.com/package/https-proxy-agent), check [its repo](https://github.com/TooTallNate/proxy-agents). - -By default, there is no request timeout. You can use the option `signal` to cancel request at the right time. +const result = await extract('https://www.cnbc.com/2022/09/21/what-another-major-rate-hike-by-the-federal-reserve-means-to-you.html', {}, myFetcher) +``` -The common way is to use AbortControler: +**Custom headers**: ```js -const controller = new AbortController() - -// stop after 5 seconds -setTimeout(() => { - controller.abort() -}, 5000) - -const data = await extract(url, null, { - signal: controller.signal, +const myFetcher = (url) => fetch(url, { + headers: { + 'user-agent': 'Opera/9.60 (Windows NT 6.0; U; en) Presto/2.1.1', + 'authorization': 'Bearer token123', + }, }) + +const result = await extract(url, {}, myFetcher) ``` -A newer solution is AbortSignal's `timeout()` static method: +**Request timeout**: ```js -// stop after 5 seconds -const data = await extract(url, null, { +const myFetcher = (url) => fetch(url, { signal: AbortSignal.timeout(5000), }) + +const result = await extract(url, {}, myFetcher) ``` For more info: diff --git a/SECURITY.md b/SECURITY.md index 921d17a..729e727 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -12,6 +12,6 @@ Description above is a general rule and may be altered on case by case basis. You can report low severity vulnerabilities as GitHub issues. -More severe vulnerabilities should be reported to email extractus@pwshub.com. +More severe vulnerabilities should be reported to email extractus.security@aslot.win. --- diff --git a/index.d.ts b/index.d.ts index 9cb8252..27ff96a 100644 --- a/index.d.ts +++ b/index.d.ts @@ -29,28 +29,12 @@ export interface ParserOptions { } /** - * Proxy configuration for fetching articles. - */ -export interface ProxyConfig { - /** Proxy endpoint URL */ - target?: string - /** Headers for proxy request */ - headers?: Record -} - -/** - * Options for the HTTP fetch request. + * Custom fetch function signature. + * + * @param url - URL to fetch + * @returns Promise resolving to a Response object */ -export interface FetchOptions { - /** Custom request headers */ - headers?: Record - /** Proxy configuration */ - proxy?: ProxyConfig - /** HTTP proxy agent (e.g. HttpsProxyAgent) */ - agent?: object - /** AbortSignal to cancel the request */ - signal?: object -} +export type Fetcher = (url: string) => Promise /** * Extracted article data structure. @@ -116,13 +100,13 @@ export function setSanitizeHtmlOptions (options: SanitizeOptions): void * * @param input - URL or HTML string to extract from * @param parserOptions - Options for parsing - * @param fetchOptions - Options for HTTP fetch + * @param fetcher - Custom fetch function (url) => Promise. Defaults to globalThis.fetch. * @returns Extracted article data or null */ export function extract ( input: string, parserOptions?: ParserOptions, - fetchOptions?: FetchOptions, + fetcher?: Fetcher, ): Promise /** diff --git a/package.json b/package.json index 14fb989..31a7a29 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,5 @@ { - "version": "8.1.0", + "version": "9.0.0-rc0", "name": "@extractus/article-extractor", "description": "To extract main article from given URL", "homepage": "https://github.com/extractus/article-extractor", @@ -31,17 +31,17 @@ }, "dependencies": { "@mozilla/readability": "^0.6.0", - "@pwshub/bellajs": "^13.0.2", - "linkedom": "^0.18.12", - "sanitize-html": "^2.17.3" + "@pwshub/bellajs": "^13.1.0", + "linkedom": "^0.18.13", + "sanitize-html": "^2.17.6" }, "devDependencies": { "@eslint/js": "^10.0.1", "@types/sanitize-html": "^2.16.1", - "eslint": "^10.3.0", - "globals": "^17.6.0", - "https-proxy-agent": "^9.0.0", - "nock": "^14.0.14" + "eslint": "^10.8.0", + "globals": "^17.9.0", + "https-proxy-agent": "^9.1.0", + "nock": "^14.0.17" }, "files": [ "src", diff --git a/src/main.js b/src/main.js index bffaf4a..4142c84 100644 --- a/src/main.js +++ b/src/main.js @@ -13,11 +13,11 @@ import { isValid as isValidUrl } from './utils/linker.js' * Load and extract article data from a URL or HTML string. * * @param {string} input - URL or HTML string to extract from - * @param {ParserOptions} [parserOptions={}] - Options for parsing - * @param {FetchOptions} [fetchOptions={}] - Options for HTTP fetch - * @returns {Promise} Extracted article data or null + * @param {object} [parserOptions={}] - Options for parsing + * @param {Function} [fetcher] - Custom fetch function (url) => Promise. Defaults to globalThis.fetch. + * @returns {Promise} Extracted article data or null */ -export const extract = async (input, parserOptions = {}, fetchOptions = {}) => { +export const extract = async (input, parserOptions = {}, fetcher = globalThis.fetch) => { if (!isString(input)) { throw new Error('Input must be a string') } @@ -25,7 +25,7 @@ export const extract = async (input, parserOptions = {}, fetchOptions = {}) => { if (!isValidUrl(input)) { return parseFromHtml(input, null, parserOptions) } - const buffer = await retrieve(input, fetchOptions) + const buffer = await retrieve(input, fetcher) const text = buffer ? Buffer.from(buffer).toString().trim() : '' if (!text) { return null diff --git a/src/main.test.js b/src/main.test.js index b52794f..a9251f0 100644 --- a/src/main.test.js +++ b/src/main.test.js @@ -5,8 +5,6 @@ import assert from 'node:assert' import { readFileSync } from 'fs' -import { HttpsProxyAgent } from 'https-proxy-agent' - import nock from 'nock' import { @@ -151,12 +149,14 @@ describe('test extract with modified sanitize-html options', () => { }) if (PROXY_SERVER !== '') { - describe('test extract live article API via proxy server', () => { - it('check if extract method works with proxy server', async () => { + describe('test extract live article API via custom fetcher', () => { + it('check if extract method works with custom fetcher', async () => { + const { HttpsProxyAgent } = await import('https-proxy-agent') const url = 'https://www.cnbc.com/2022/09/21/what-another-major-rate-hike-by-the-federal-reserve-means-to-you.html' - const result = await extract(url, {}, { + const myFetcher = (fetchUrl) => fetch(fetchUrl, { agent: new HttpsProxyAgent(PROXY_SERVER), }) + const result = await extract(url, {}, myFetcher) assert.ok(result.title.includes('Federal Reserve')) assert.equal(result.source, 'cnbc.com') }, 10000) diff --git a/src/utils/retrieve.js b/src/utils/retrieve.js index 06f1e59..a00d8c4 100644 --- a/src/utils/retrieve.js +++ b/src/utils/retrieve.js @@ -1,48 +1,14 @@ // utils -> retrieve /** - * Fetch content through a proxy endpoint. - * - * @param {string} url - Target URL to fetch - * @param {Object} [options={}] - Proxy options - * @param {Object} [options.proxy={}] - Proxy configuration - * @param {string} options.proxy.target - Proxy endpoint URL - * @param {Object} [options.proxy.headers={}] - Headers for proxy request - * @param {AbortSignal} [options.signal] - Optional abort signal - * @returns {Promise} Fetch response object - */ -const profetch = async (url, options = {}) => { - const { proxy = {}, signal = null } = options - const { - target, - headers = {}, - } = proxy - const res = await fetch(target + encodeURIComponent(url), { - headers, - signal, - }) - return res -} - -/** - * Retrieve raw HTML content from a URL. - * Supports direct fetch, proxy, custom headers, agent, and abort signal. + * Fetch content from a URL using the provided fetcher function. * * @param {string} url - URL to fetch - * @param {FetchOptions} [options={}] - Fetch configuration - * @returns {Promise} Response body as ArrayBuffer + * @param {Function} fetcher - Custom fetch function (url) => Promise + * @returns {Promise} Response body as ArrayBuffer */ -export default async (url, options = {}) => { - const { - headers = { - 'user-agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/115.0', - }, - proxy = null, - agent = null, - signal = null, - } = options - - const res = proxy ? await profetch(url, { proxy, signal }) : await fetch(url, { headers, agent, signal }) +export default async (url, fetcher) => { + const res = await fetcher(url) const status = res.status if (status >= 400) { diff --git a/src/utils/retrieve.test.js b/src/utils/retrieve.test.js index a410c00..0e95e3b 100644 --- a/src/utils/retrieve.test.js +++ b/src/utils/retrieve.test.js @@ -19,7 +19,7 @@ describe('test retrieve() method', () => { const url = 'https://some.where/bad/page' const { baseUrl, path } = parseUrl(url) nock(baseUrl).get(path).reply(500, 'Error 500') - assert.rejects(retrieve(url), new Error('Request failed with error code 500')) + assert.rejects(retrieve(url, globalThis.fetch), new Error('Request failed with error code 500')) }) it('test retrieve from good source', async () => { @@ -28,7 +28,7 @@ describe('test retrieve() method', () => { nock(baseUrl).get(path).reply(200, '
this is content
', { 'Content-Type': 'text/html', }) - const buffer = await retrieve(url) + const buffer = await retrieve(url, globalThis.fetch) const html = Buffer.from(buffer).toString() assert.equal(html, '
this is content
') }) @@ -39,28 +39,25 @@ describe('test retrieve() method', () => { nock(baseUrl).get(path).reply(200, '\n\r\r\n\n
this is content
\n\r\r\n\n', { 'Content-Type': 'text/html', }) - const buffer = await retrieve(url) + const buffer = await retrieve(url, globalThis.fetch) const html = Buffer.from(buffer).toString().trim() assert.equal(html, '
this is content
') }) - it('test retrieve using proxy', async () => { - const url = 'https://some.where/good/source-with-proxy' - const { baseUrl, path } = parseUrl(url) - nock(baseUrl).get(path).reply(200, 'something bad', { - 'Content-Type': 'bad/thing', - }) + it('test retrieve with custom fetcher', async () => { + const url = 'https://some.where/good/source-with-fetcher' nock('https://proxy-server.com') - .get('/api/proxy?url=https%3A%2F%2Fsome.where%2Fgood%2Fsource-with-proxy') + .get('/api/proxy?url=https%3A%2F%2Fsome.where%2Fgood%2Fsource-with-fetcher') .reply(200, '
this is content
', { 'Content-Type': 'text/html', }) - const buffer = await retrieve(url, { - proxy: { - target: 'https://proxy-server.com/api/proxy?url=', - }, - }) + const myFetcher = (fetchUrl) => { + const proxyUrl = 'https://proxy-server.com/api/proxy?url=' + encodeURIComponent(fetchUrl) + return globalThis.fetch(proxyUrl) + } + + const buffer = await retrieve(url, myFetcher) const html = Buffer.from(buffer).toString() assert.equal(html, '
this is content
') nock.cleanAll() From 9d91b1195b03863cb41621d1035a55792b35225c Mon Sep 17 00:00:00 2001 From: Dong Nguyen Date: Thu, 6 Aug 2026 14:47:59 +0700 Subject: [PATCH 2/6] Fix logic at JSON-LD metadata processing Related issue: #425 --- src/utils/extractLdSchema.js | 2 +- src/utils/extractMetaData.test.js | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/utils/extractLdSchema.js b/src/utils/extractLdSchema.js index 2f9584b..d4cc82d 100644 --- a/src/utils/extractLdSchema.js +++ b/src/utils/extractLdSchema.js @@ -90,7 +90,7 @@ export default (document, entry) => { const ldJson = parseJson(ldSchema.textContent.replace(/[\n\r\t]/g, '')) if (ldJson && isAllowedLdJsonType(ldJson)) { for (const [key, attr] of Object.entries(attributeLists)) { - if (!entry[key] || !ldJson[attr]) { + if (entry[key] || !ldJson[attr]) { continue } diff --git a/src/utils/extractMetaData.test.js b/src/utils/extractMetaData.test.js index 00fec7c..dd0e681 100644 --- a/src/utils/extractMetaData.test.js +++ b/src/utils/extractMetaData.test.js @@ -35,6 +35,27 @@ describe('test extractMetaData', () => { }) }) + it('test extractMetaData(json ld schema fills empty meta fields)', async () => { + const ldJson = { + '@context': 'https://schema.org', + '@type': 'NewsArticle', + author: { name: 'JSON Author' }, + datePublished: '2026-01-02T03:04:05Z', + description: 'JSON description', + image: 'https://example.com/json.jpg', + } + const body = '

' + 'Article text. '.repeat(50) + '

' + const html = ` +Fixture article + +

Fixture article

${body}
` + const result = extractMetaData(html) + assert.strictEqual(result.author, 'JSON Author') + assert.strictEqual(result.published, '2026-01-02T03:04:05Z') + assert.strictEqual(result.description, 'JSON description') + assert.strictEqual(result.image, 'https://example.com/json.jpg') + }) + it('test extractMetaData(find date)', async () => { const html1 = readFileSync('./test-data/regular-article-date-time.html', 'utf8') const html2 = readFileSync('./test-data/regular-article-date-itemprop.html', 'utf8') From 32476b724fe8058a09eb9bc203121ae7301845dc Mon Sep 17 00:00:00 2001 From: Dong Nguyen Date: Thu, 6 Aug 2026 16:27:09 +0700 Subject: [PATCH 3/6] v9.0.0 - Deno/JSR first --- .aiignore | 15 - .github/workflows/ci-test.yml | 37 +- .github/workflows/codeql-analysis.yml | 70 --- .github/workflows/release.yml | 32 ++ .gitignore | 23 +- .npmignore | 10 - AGENTS.md | 113 ---- CONTRIBUTING.md | 28 +- README.md | 485 +++++++----------- deno.json | 49 ++ deno.lock | 268 ++++++++++ eslint.config.js | 127 ----- eval.js | 53 -- index.d.ts | 124 ----- mod.ts | 14 + package.json | 59 --- reset.js | 34 -- scripts/build_npm.ts | 39 ++ src/config.js | 80 --- src/config.test.js | 37 -- src/config.ts | 65 +++ src/main.js | 52 -- src/main.test.js | 164 ------ src/main.ts | 112 ++++ src/utils/extractLdSchema.js | 107 ---- src/utils/extractLdSchema.ts | 77 +++ src/utils/extractMetaData.js | 154 ------ src/utils/extractMetaData.test.js | 78 --- src/utils/extractMetaData.ts | 148 ++++++ src/utils/extractWithReadability.js | 42 -- src/utils/extractWithReadability.test.js | 43 -- src/utils/extractWithReadability.ts | 29 ++ src/utils/findDate.js | 59 --- src/utils/findDate.ts | 48 ++ src/utils/html.js | 100 ---- src/utils/html.test.js | 23 - src/utils/html.ts | 57 ++ src/utils/linker.js | 185 ------- src/utils/linker.test.js | 182 ------- src/utils/linker.ts | 135 +++++ src/utils/parseFromHtml.js | 163 ------ src/utils/parseFromHtml.test.js | 133 ----- src/utils/parseFromHtml.ts | 143 ++++++ src/utils/retrieve.js | 19 - src/utils/retrieve.test.js | 65 --- src/utils/retrieve.ts | 12 + src/utils/similarity.js | 48 -- src/utils/similarity.ts | 45 ++ src/utils/transformation.js | 122 ----- src/utils/transformation.test.js | 194 ------- src/utils/transformation.ts | 83 +++ tests/config.test.ts | 31 ++ tests/extractMetaData.test.ts | 72 +++ tests/extractWithReadability.test.ts | 36 ++ tests/helpers.ts | 15 + tests/html.test.ts | 15 + tests/linker.test.ts | 69 +++ tests/main.test.ts | 93 ++++ tests/parseFromHtml.test.ts | 96 ++++ tests/retrieve.test.ts | 43 ++ .../article-with-classes-attributes.html | 0 .../test-data}/html-article-no-source.html | 0 .../html-article-with-data-src.html | 0 .../test-data}/html-no-article.html | 0 .../test-data}/html-no-link.html | 0 .../test-data}/html-no-title.html | 0 .../test-data}/html-too-short-article.html | 0 .../regular-article-date-itemprop.html | 0 .../test-data}/regular-article-date-span.html | 0 .../test-data}/regular-article-date-time.html | 0 .../test-data}/regular-article-json-ld.html | 0 .../test-data}/regular-article.html | 0 .../test-data}/vnn-article.html | 0 tests/transformation.test.ts | 191 +++++++ 74 files changed, 2221 insertions(+), 3024 deletions(-) delete mode 100644 .aiignore delete mode 100644 .github/workflows/codeql-analysis.yml create mode 100644 .github/workflows/release.yml delete mode 100644 .npmignore delete mode 100644 AGENTS.md create mode 100644 deno.json create mode 100644 deno.lock delete mode 100644 eslint.config.js delete mode 100644 eval.js delete mode 100644 index.d.ts create mode 100644 mod.ts delete mode 100644 package.json delete mode 100644 reset.js create mode 100644 scripts/build_npm.ts delete mode 100644 src/config.js delete mode 100644 src/config.test.js create mode 100644 src/config.ts delete mode 100644 src/main.js delete mode 100644 src/main.test.js create mode 100644 src/main.ts delete mode 100644 src/utils/extractLdSchema.js create mode 100644 src/utils/extractLdSchema.ts delete mode 100644 src/utils/extractMetaData.js delete mode 100644 src/utils/extractMetaData.test.js create mode 100644 src/utils/extractMetaData.ts delete mode 100644 src/utils/extractWithReadability.js delete mode 100644 src/utils/extractWithReadability.test.js create mode 100644 src/utils/extractWithReadability.ts delete mode 100644 src/utils/findDate.js create mode 100644 src/utils/findDate.ts delete mode 100644 src/utils/html.js delete mode 100644 src/utils/html.test.js create mode 100644 src/utils/html.ts delete mode 100644 src/utils/linker.js delete mode 100644 src/utils/linker.test.js create mode 100644 src/utils/linker.ts delete mode 100644 src/utils/parseFromHtml.js delete mode 100644 src/utils/parseFromHtml.test.js create mode 100644 src/utils/parseFromHtml.ts delete mode 100644 src/utils/retrieve.js delete mode 100644 src/utils/retrieve.test.js create mode 100644 src/utils/retrieve.ts delete mode 100644 src/utils/similarity.js create mode 100644 src/utils/similarity.ts delete mode 100644 src/utils/transformation.js delete mode 100644 src/utils/transformation.test.js create mode 100644 src/utils/transformation.ts create mode 100644 tests/config.test.ts create mode 100644 tests/extractMetaData.test.ts create mode 100644 tests/extractWithReadability.test.ts create mode 100644 tests/helpers.ts create mode 100644 tests/html.test.ts create mode 100644 tests/linker.test.ts create mode 100644 tests/main.test.ts create mode 100644 tests/parseFromHtml.test.ts create mode 100644 tests/retrieve.test.ts rename {test-data => tests/test-data}/article-with-classes-attributes.html (100%) rename {test-data => tests/test-data}/html-article-no-source.html (100%) rename {test-data => tests/test-data}/html-article-with-data-src.html (100%) rename {test-data => tests/test-data}/html-no-article.html (100%) rename {test-data => tests/test-data}/html-no-link.html (100%) rename {test-data => tests/test-data}/html-no-title.html (100%) rename {test-data => tests/test-data}/html-too-short-article.html (100%) rename {test-data => tests/test-data}/regular-article-date-itemprop.html (100%) rename {test-data => tests/test-data}/regular-article-date-span.html (100%) rename {test-data => tests/test-data}/regular-article-date-time.html (100%) rename {test-data => tests/test-data}/regular-article-json-ld.html (100%) rename {test-data => tests/test-data}/regular-article.html (100%) rename {test-data => tests/test-data}/vnn-article.html (100%) create mode 100644 tests/transformation.test.ts diff --git a/.aiignore b/.aiignore deleted file mode 100644 index c880c25..0000000 --- a/.aiignore +++ /dev/null @@ -1,15 +0,0 @@ -node_modules -coverage -coverage.lcov - -package-lock.json -pnpm-lock.yaml -bun.lock - -.env - -dist -storage - -# AI Session Files (Private Context) -.sessions diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index d6c6a5f..ac7a6c4 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -1,38 +1,27 @@ -# GitHub actions -# https://docs.github.com/en/free-pro-team@latest/actions - name: ci-test on: [push, pull_request] jobs: test: - runs-on: ubuntu-latest - strategy: matrix: - node_version: [22.x, 24.x, 25.x] + deno-version: ["lts", "canary"] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 + + - name: Setup Deno + uses: denoland/setup-deno@v2 + with: + deno-version: ${{ matrix.deno-version }} - - name: setup Node.js v${{ matrix.node_version }} - uses: actions/setup-node@v6 - with: - node-version: ${{ matrix.node_version }} + - name: lint + run: deno lint - - name: run npm scripts - run: | - npm install - npm run lint - #npm run build --if-present - npm run test + - name: test + run: deno test --allow-all - - name: cache node modules - uses: actions/cache@v5 - with: - path: ~/.npm - key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} - restore-keys: | - ${{ runner.os }}-node- + - name: audit + run: deno audit diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml deleted file mode 100644 index 5547051..0000000 --- a/.github/workflows/codeql-analysis.yml +++ /dev/null @@ -1,70 +0,0 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# -name: "CodeQL" - -on: - push: - branches: [ main ] - pull_request: - # The branches below must be a subset of the branches above - branches: [ main ] - schedule: - - cron: '33 18 * * 3' - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - permissions: - actions: read - contents: read - security-events: write - - strategy: - fail-fast: false - matrix: - language: [ 'javascript' ] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] - # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support - - steps: - - name: Checkout repository - uses: actions/checkout@v6 - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main - - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v2 - - # â„šī¸ Command-line programs to run using the OS shell. - # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun - - # âœī¸ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language - - #- run: | - # make bootstrap - # make release - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..028f8df --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,32 @@ +name: Release + +on: + release: + types: [published] + +jobs: + publish: + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + + steps: + - uses: actions/checkout@v7 + + - name: Setup Deno + uses: denoland/setup-deno@v2 + with: + deno-version: lts + + - name: lint + run: deno lint + + - name: test + run: deno test --allow-all + + - name: audit + run: deno audit + + - name: publish to jsr + run: deno publish diff --git a/.gitignore b/.gitignore index 971acbc..3a7c626 100644 --- a/.gitignore +++ b/.gitignore @@ -1,24 +1,5 @@ -# Logs -logs -*.log -*.debug - -# Runtime data -*.pid -*.seed - node_modules -coverage -.nyc_output - -.DS_Store -yarn.lock -coverage.lcov -pnpm-lock.yaml -lcov.info -bun.lock -deno.lock -evaluation +.env -.sessions +npm/ diff --git a/.npmignore b/.npmignore deleted file mode 100644 index a0f6f85..0000000 --- a/.npmignore +++ /dev/null @@ -1,10 +0,0 @@ -node_modules -coverage -.github -pnpm-lock.yaml -examples -test-data -lcov.info - -.aiignore -.sessions diff --git a/AGENTS.md b/AGENTS.md deleted file mode 100644 index bfc44e1..0000000 --- a/AGENTS.md +++ /dev/null @@ -1,113 +0,0 @@ -# AI Agent Instructions - -Coding guidelines for AI agents working in this project. - -## Philosophy - -- Minimalism. Simple is better. KISS (Keep It Simple, Stupid). -- Clean code, easy to read, easy to delete. -- Functional Programming — pure functions, immutability, no side effects. -- MVP mindset — deliver the smallest thing that works, then iterate. - -## Security Rules (CRITICAL — no exceptions) - -- NEVER output or request .env and example.env file contents -- NEVER hardcode API credentials, secret tokens, private keys or passwords in source code -- NEVER send sensitive data to external AI services -- Follow `.aiignore` and `.gitignore` for excluded files — do not read or reference them -- When asking for help, sanitize data (replace real IDs, emails, tokens with placeholders) -- Do not log sensitive information - -## Coding Standards (Strict) -- Language: JavaScript (ESM syntax). No TypeScript. -- Style: No semicolons, single quotes, 2-space indentation. -- Respect `eslint.config.js` — do not suggest rule changes -- Patterns: - - Functional Programming only. No Classes or OOP. - - Arrow functions are preferred. - - Maximum 3 parameters per function. Use objects for more. -- Naming: camelCase for variables/functions, SNAKE_CASE for constants. -- Documentation: - - Add JSDocs before all functions and exported variables. - - Language: Use American English for all comments and JSDocs. - - Constraint: NEVER use Vietnamese or other languages in the source code. - -### Error Handling - -- Handle errors explicitly — never swallow silently -- Use try/catch with proper logging -- Return null or throw meaningful errors - -```javascript -export const send = async (params) => { - try { - const response = await ai.ask(params) - logger.info(`send() -> success: ${response.id}`) - return response - } catch (err) { - logger.error(`send() -> failed: ${err.message}`) - console.error(err) - return null - } -} -``` - -## Testing Standards - -- Write tests for critical business logic, all error cases -- Use simple test runners (node:test, bun:test, vitest) -- No complex mocking frameworks unless necessary -- Tests live alongside source: `[module].test.js` next to `[module].js` - -## Dependency Rules - -- Prefer built-in APIs over external packages -- Before adding dependency, explain: - - Why it is needed - - Alternatives considered - - Bundle size impact -- Never add dependency for trivial utilities -- Avoid packages with large dependency trees - -## Architecture Rules - -- Do NOT change existing project architecture without explicit approval -- Do NOT move or rename core modules unless requested -- Respect module boundaries -- Avoid cross-module coupling -- New modules must follow existing folder structure - -## When Making Changes - -1. Read existing patterns first -2. Follow current coding style strictly -3. Keep dependencies minimal -4. Handle errors explicitly -5. Add JSDoc comments for new functions -6. Run `npm run lint` before committing -7. Do NOT refactor unrelated code -8. Do NOT modify working code outside task scope -9. Prefer minimal diff changes -10. Preserve existing behavior unless explicitly requested - -## When in Doubt - -- Ask for clarification before generating code -- State your assumption explicitly if proceeding without confirmation -- Prefer doing less and asking over doing more and guessing - -## Git Workflow - -- Work only inside the current branch -- Do NOT create or delete branches -- Do NOT rewrite git history -- Do NOT modify commit messages -- Changes must correspond to the current issue - -## Agent References - -Reference these URLs when working on related topics: - -- Bun: https://bun.sh/llms-full.txt - ---- diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8cfca77..c28b4db 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,8 +10,7 @@ As an OSS, it's better to follow the Unix philosophy: "do one thing and do it we Please avoid using libaries other than those available in the standard library, unless necessary. -This library needs to be simple and flexible to run on multiple platforms such as Deno, Bun, or even browser. - +This library needs to be simple and flexible to run on multiple platforms (Deno, Node, Bun). ## Coding convention @@ -22,34 +21,21 @@ Make sure your code lints before opening a pull request. cd article-extractor # check coding convention issue -npm run lint +deno lint # auto fix coding convention issue -npm run lint:fix +deno lint --fix ``` -*When you run `npm test`, the linting process will be triggered at first.* - - ## Testing Be sure to run the unit test suite before opening a pull request. An example test run is shown below. ```bash cd article-extractor -npm test +deno test --allow-all ``` -![article-extractor unit test](https://i.imgur.com/TbRCUSS.png?110222) - -If test coverage decreased, please check test scripts and try to improve this number. - - -## Documentation - -If you've changed APIs, please update README and [the examples](examples). - - ## Clean commit histories When you open a pull request, please ensure the commit history is clean. @@ -58,12 +44,6 @@ Squash the commits into logical blocks, perhaps a single commit if that makes se What you want to avoid is commits such as "WIP" and "fix test" in the history. This is so we keep history on master clean and straightforward. -For people new to git, please refer the following guides: - -- [Writing good commit messages](https://github.com/erlang/otp/wiki/writing-good-commit-messages) -- [Commit Message Guidelines](https://gist.github.com/robertpainsi/b632364184e70900af4ab688decf6f53) - - ## License By contributing to `@extractus/article-extractor`, you agree that your contributions will be licensed under its [MIT license](LICENSE). diff --git a/README.md b/README.md index 5215ffc..f6b6e5f 100644 --- a/README.md +++ b/README.md @@ -2,128 +2,122 @@ Extract main article, main image and meta data from URL. +[![JSR](https://jsr.io/badges/@extractus/article-extractor)](https://jsr.io/@extractus/article-extractor) [![npm version](https://badge.fury.io/js/@extractus%2Farticle-extractor.svg)](https://badge.fury.io/js/@extractus%2Farticle-extractor) -![CodeQL](https://github.com/extractus/article-extractor/workflows/CodeQL/badge.svg) ![CI test](https://github.com/extractus/article-extractor/workflows/ci-test/badge.svg) -(This library is derived from [article-parser](https://www.npmjs.com/package/article-parser) renamed.) +## Install -## Demo +### Deno -- [Give it a try!](https://extractus.pwshub.com/article) +```bash +deno add jsr:@extractus/article-extractor +``` -## Install +### Node.js / Bun ```bash -# bun -bun add @extractus/article-extractor - -# npm -npm i @extractus/article-extractor +pnpm add jsr:@extractus/article-extractor +# or +npx jsr add @extractus/article-extractor +# or +bunx jsr add @extractus/article-extractor +``` -# pnpm -pnpm install @extractus/article-extractor +Alternatively, install from npm: -# yarn -yarn add @extractus/article-extractor +```bash +npm install @extractus/article-extractor +# or +bun add @extractus/article-extractor ``` ## Usage ```ts -import { extract } from '@extractus/article-extractor' +import { extract } from "jsr:@extractus/article-extractor"; -const data = await extract(ARTICLE_URL) -console.log(data) +const data = await extract("https://example.com/article"); +console.log(data); ``` ## APIs -- [extract()](#extract) -- [extractFromHtml()](#extractfromhtml) +- [`extract()`](#extract) +- [`extractFromHtml()`](#extractfromhtml) - [Transformations](#transformations) - - [`transformation` object](#transformation-object) - - [.addTransformations](#addtransformationsobject-transformation--array-transformations) - - [.removeTransformations](#removetransformationsarray-patterns) + - [`Transformation` object](#transformation-object) + - [`addTransformations()`](#addtransformationstransformation--transformation) + - [`removeTransformations()`](#removetransformationspatterns-regexp) - [Priority order](#priority-order) -- [`sanitize-html`'s options](#sanitize-htmls-options) +- [`sanitize-html` options](#sanitize-html-options) --- ### `extract()` -Load and extract article data. Return a Promise object. +Load and extract article data from a URL or HTML string. #### Syntax ```ts -extract(String input) -extract(String input, Object parserOptions) -extract(String input, Object parserOptions, Function fetcher) +extract(input: string): Promise +extract(input: string, parserOptions?: ParserOptions): Promise +extract(input: string, parserOptions?: ParserOptions, fetcher?: Fetcher): Promise ``` Example: -```js -import { extract } from '@extractus/article-extractor' - -const input = 'https://www.cnbc.com/2022/09/21/what-another-major-rate-hike-by-the-federal-reserve-means-to-you.html' +```ts +import { extract } from "jsr:@extractus/article-extractor"; -// here we use top-level await, assume current platform supports it try { - const article = await extract(input) - console.log(article) + const article = await extract("https://example.com/some-article"); + console.log(article); } catch (err) { - console.error(err) + console.error(err); } ``` -The result - `article` - can be `null` or an object with the following structure: +The result can be `null` (when no article found) or an `ArticleData` object: ```ts -{ - url: String, - title: String, - description: String, - image: String, - author: String, - favicon: String, - content: String, - published: Date String, - type: String, // page type - source: String, // original publisher - links: Array, // list of alternative links - ttr: Number, // time to read in second, 0 = unknown +interface ArticleData { + url?: string; // best resolved URL + links?: string[]; // alternative URLs (canonical, shortlink, amphtml) + title?: string; // article title + description?: string; // short description / excerpt + image?: string; // main image URL + favicon?: string; // site favicon URL + author?: string; // author name + content?: string; // extracted article HTML + source?: string; // original publisher domain + published?: string; // publication date string + ttr?: number; // estimated time to read (seconds), 0 = unknown + type?: string; // page type (e.g. "article") } ``` - #### Parameters ##### `input` *required* -URL string links to the article or HTML content of that web page. +URL string or raw HTML content. ##### `parserOptions` *optional* -Object with all or several of the following properties: +| Property | Type | Default | Description | +|---|---|---|---| +| `wordsPerMinute` | `number` | `300` | Words per minute for time-to-read estimation | +| `descriptionTruncateLen` | `number` | `210` | Max characters for generated description | +| `descriptionLengthThreshold` | `number` | `180` | Min characters to keep meta description | +| `contentLengthThreshold` | `number` | `200` | Min characters for article content | - - `wordsPerMinute`: Number, to estimate time to read. Default `300`. - - `descriptionTruncateLen`: Number, max num of chars generated for description. Default `210`. - - `descriptionLengthThreshold`: Number, min num of chars required for description. Default `180`. - - `contentLengthThreshold`: Number, min num of chars required for content. Default `200`. - -For example: - -```js -import { extract } from '@extractus/article-extractor' - -const article = await extract('https://www.cnbc.com/2022/09/21/what-another-major-rate-hike-by-the-federal-reserve-means-to-you.html', { +```ts +const article = await extract(url, { descriptionLengthThreshold: 120, - contentLengthThreshold: 500 -}) - -console.log(article) + contentLengthThreshold: 500, +}); ``` ##### `fetcher` *optional* @@ -133,349 +127,240 @@ Use this to customize HTTP behavior: proxy, headers, TLS, authentication, timeou Defaults to `globalThis.fetch`. -**Node.js** (with proxy via undici): +**Deno** (with proxy): -```js -import { extract } from '@extractus/article-extractor' -import { fetch, ProxyAgent } from 'undici' +```ts +import { extract } from "@extractus/article-extractor"; -const dispatcher = new ProxyAgent('http://proxy.example.com:8080') -const myFetcher = (url) => fetch(url, { dispatcher }) +const client = Deno.createHttpClient({ + proxy: { url: "http://proxy.example.com:8080" }, +}); +const myFetcher = (url: string) => fetch(url, { client }); -const result = await extract('https://www.cnbc.com/2022/09/21/what-another-major-rate-hike-by-the-federal-reserve-means-to-you.html', {}, myFetcher) +const result = await extract("https://example.com/some-article", {}, myFetcher); ``` -**Bun** (with proxy): +**Node.js** (with proxy via undici): -```js -import { extract } from '@extractus/article-extractor' +```ts +import { extract } from "@extractus/article-extractor"; +import { fetch, ProxyAgent } from "undici"; -const myFetcher = (url) => fetch(url, { - proxy: { - url: 'http://proxy.example.com:8080', - }, -}) +const dispatcher = new ProxyAgent("http://proxy.example.com:8080"); +const myFetcher = (url: string) => fetch(url, { dispatcher }); -const result = await extract('https://www.cnbc.com/2022/09/21/what-another-major-rate-hike-by-the-federal-reserve-means-to-you.html', {}, myFetcher) +const result = await extract("https://example.com/some-article", {}, myFetcher); ``` -**Deno** (with proxy): +**Bun** (with proxy): -```js -import { extract } from 'npm:@extractus/article-extractor' +```ts +import { extract } from "@extractus/article-extractor"; -const client = Deno.createHttpClient({ - proxy: { url: 'http://localhost:8080' }, -}) -const myFetcher = (url) => fetch(url, { client }) +const myFetcher = (url: string) => + fetch(url, { + proxy: "http://proxy.example.com:8080", + }); -const result = await extract('https://www.cnbc.com/2022/09/21/what-another-major-rate-hike-by-the-federal-reserve-means-to-you.html', {}, myFetcher) +const result = await extract("https://example.com/some-article", {}, myFetcher); ``` **Custom headers**: -```js -const myFetcher = (url) => fetch(url, { - headers: { - 'user-agent': 'Opera/9.60 (Windows NT 6.0; U; en) Presto/2.1.1', - 'authorization': 'Bearer token123', - }, -}) +```ts +const myFetcher = (url: string) => + fetch(url, { + headers: { + "user-agent": "MyBot/1.0", + authorization: "Bearer token123", + }, + }); -const result = await extract(url, {}, myFetcher) +const result = await extract("https://example.com/some-article", {}, myFetcher); ``` **Request timeout**: -```js -const myFetcher = (url) => fetch(url, { - signal: AbortSignal.timeout(5000), -}) +```ts +const myFetcher = (url: string) => + fetch(url, { + signal: AbortSignal.timeout(5000), + }); -const result = await extract(url, {}, myFetcher) +const result = await extract("https://example.com/some-article", {}, myFetcher); ``` - -For more info: - -- [AbortController constructor](https://developer.mozilla.org/en-US/docs/Web/API/AbortController) -- [AbortSignal: timeout() static method](https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal/timeout_static) - +--- ### `extractFromHtml()` -Extract article data from HTML string. Return a Promise object as same as `extract()` method above. +Extract article data from an HTML string directly. #### Syntax ```ts -extractFromHtml(String html) -extractFromHtml(String html, String url) -extractFromHtml(String html, String url, Object parserOptions) +extractFromHtml(html: string): Promise +extractFromHtml(html: string, url?: string): Promise +extractFromHtml(html: string, url?: string, parserOptions?: ParserOptions): Promise ``` Example: -```js -import { extractFromHtml } from '@extractus/article-extractor' - -const url = 'https://www.cnbc.com/2022/09/21/what-another-major-rate-hike-by-the-federal-reserve-means-to-you.html' - -const res = await fetch(url) -const html = await res.text() +```ts +import { extractFromHtml } from "jsr:@extractus/article-extractor"; -// you can do whatever with this raw html here: clean up, remove ads banner, etc -// just ensure a html string returned +const res = await fetch(url); +const html = await res.text(); -const article = await extractFromHtml(html, url) -console.log(article) +const article = await extractFromHtml(html, url); ``` #### Parameters ##### `html` *required* -HTML string which contains the article you want to extract. +HTML string containing the article. ##### `url` *optional* -URL string that indicates the source of that HTML content. -`article-extractor` may use this info to handle internal/relative links. +Source URL for resolving relative links. ##### `parserOptions` *optional* See [parserOptions](#parseroptions-optional) above. - --- ### Transformations -Sometimes the default extraction algorithm may not work well. That is the time when we need transformations. - -By adding some functions before and after the main extraction step, we aim to come up with a better result as much as possible. - -There are 2 methods to play with transformations: - -- `addTransformations(Object transformation | Array transformations)` -- `removeTransformations(Array patterns)` - -At first, let's talk about `transformation` object. +Sometimes the default extraction algorithm may not work well. Transformations let you add pre/post processing per-site. -#### `transformation` object +- `addTransformations(transformation: Transformation | Transformation[]): number` +- `removeTransformations(patterns?: RegExp[]): number` -In `@extractus/article-extractor`, `transformation` is an object with the following properties: +#### `Transformation` object -- `patterns`: required, a list of regexps to match the URLs -- `pre`: optional, a function to process raw HTML -- `post`: optional, a function to process extracted article - -Basically, the meaning of `transformation` can be interpreted like this: +```ts +interface Transformation { + patterns: RegExp[]; // URL patterns to match + pre?: (document: Document) => Document; // pre-process raw HTML + post?: (document: Document) => Document; // post-process extracted article +} +``` -> with the urls which match these `patterns`
-> let's run `pre` function to normalize HTML content
-> then extract main article content with normalized HTML, and if success
-> let's run `post` function to normalize extracted article content +> For URLs matching `patterns`, run `pre` on raw HTML, extract article, then run `post` on the result. -![article-extractor extraction process](https://res.cloudinary.com/pwshub/image/upload/v1657336822/documentation/article-parser_extraction_process.png) +![extraction process](https://res.cloudinary.com/pwshub/image/upload/v1657336822/documentation/article-parser_extraction_process.png) -Here is an example transformation: +Example: ```ts -{ - patterns: [ - /([\w]+.)?domain.tld\/*/, - /domain.tld\/articles\/*/ - ], +import { addTransformations } from "jsr:@extractus/article-extractor"; + +addTransformations({ + patterns: [/([\w]+.)?domain\.tld\/*/], pre: (document) => { - // remove all .advertise-area and its siblings from raw HTML content - document.querySelectorAll('.advertise-area').forEach((element) => { - if (element.nodeName === 'DIV') { - while (element.nextSibling) { - element.parentNode.removeChild(element.nextSibling) - } - element.parentNode.removeChild(element) - } - }) - return document + document.querySelectorAll(".advertise-area").forEach((el) => { + el.parentNode?.removeChild(el); + }); + return document; }, post: (document) => { - // with extracted article, replace all h4 tags with h2 - document.querySelectorAll('h4').forEach((element) => { - const h2Element = document.createElement('h2') - h2Element.innerHTML = element.innerHTML - element.parentNode.replaceChild(h2Element, element) - }) - // change small sized images to original version - document.querySelectorAll('img').forEach((element) => { - const src = element.getAttribute('src') - if (src.includes('domain.tld/pics/150x120/')) { - const fullSrc = src.replace('/pics/150x120/', '/pics/original/') - element.setAttribute('src', fullSrc) - } - }) - return document - } -} + document.querySelectorAll("h4").forEach((el) => { + const h2 = document.createElement("h2"); + h2.innerHTML = el.innerHTML; + el.parentNode?.replaceChild(h2, el); + }); + return document; + }, +}); ``` -- To write better transformation logic, please refer [linkedom](https://github.com/WebReflection/linkedom) and [Document Object](https://developer.mozilla.org/en-US/docs/Web/API/Document). +To write better transformations, refer to [linkedom](https://github.com/WebReflection/linkedom) and the [Document API](https://developer.mozilla.org/en-US/docs/Web/API/Document). -#### `addTransformations(Object transformation | Array transformations)` +#### `addTransformations(transformation | Transformation[])` -Add a single transformation or a list of transformations. For example: +Add a single or multiple transformations. Transformations without `patterns` are ignored. ```ts -import { addTransformations } from '@extractus/article-extractor' - -addTransformations({ - patterns: [ - /([\w]+.)?abc.tld\/*/ - ], - pre: (document) => { - // do something with document - return document - }, - post: (document) => { - // do something with document - return document - } -}) +import { addTransformations } from "jsr:@extractus/article-extractor"; addTransformations([ { - patterns: [ - /([\w]+.)?def.tld\/*/ - ], - pre: (document) => { - // do something with document - return document - }, - post: (document) => { - // do something with document - return document - } + patterns: [/([\w]+.)?abc\.tld\/*/], + pre: (doc) => { /* ... */ return doc; }, + post: (doc) => { /* ... */ return doc; }, }, { - patterns: [ - /([\w]+.)?xyz.tld\/*/ - ], - pre: (document) => { - // do something with document - return document - }, - post: (document) => { - // do something with document - return document - } - } -]) -```` - -The transformations without `patterns` will be ignored. - -#### `removeTransformations(Array patterns)` + patterns: [/([\w]+.)?xyz\.tld\/*/], + post: (doc) => { /* ... */ return doc; }, + }, +]); +``` -To remove transformations that match the specific patterns. +#### `removeTransformations(patterns?: RegExp[])` -For example, we can remove all added transformations above: +Remove transformations matching the given patterns. Call without arguments to remove all. -```js -import { removeTransformations } from '@extractus/article-extractor' +```ts +import { removeTransformations } from "jsr:@extractus/article-extractor"; removeTransformations([ - /([\w]+.)?abc.tld\/*/, - /([\w]+.)?def.tld\/*/, - /([\w]+.)?xyz.tld\/*/ -]) + /([\w]+.)?abc\.tld\/*/, + /([\w]+.)?xyz\.tld\/*/, +]); ``` -Calling `removeTransformations()` without parameter will remove all current transformations. - #### Priority order -While processing an article, more than one transformation can be applied. +When multiple transformations match, they all execute in order. -Suppose that we have the following transformations: +Given two transformations matching `goo.gl`: -```ts -[ - { - patterns: [ - /http(s?):\/\/google.com\/*/, - /http(s?):\/\/goo.gl\/*/ - ], - pre: function_one, - post: function_two - }, - { - patterns: [ - /http(s?):\/\/goo.gl\/*/, - /http(s?):\/\/google.inc\/*/ - ], - pre: function_three, - post: function_four - } -] ``` - -As you can see, an article from `goo.gl` certainly matches both them. - -In this scenario, `@extractus/article-extractor` will execute both transformations, one by one: - -`function_one` -> `function_three` -> extraction -> `function_two` -> `function_four` +pre_one -> pre_three -> extraction -> post_two -> post_four +``` --- -### `sanitize-html`'s options - -`@extractus/article-extractor` uses [sanitize-html](https://github.com/apostrophecms/sanitize-html) to make a clean sweep of HTML content. +### `sanitize-html` options -Here is the [default options](src/config.js#L5) +Uses [sanitize-html](https://github.com/apostrophecms/sanitize-html) to clean extracted content. See the [default options](src/config.ts). -Depending on the needs of your content system, you might want to gather some HTML tags/attributes, while ignoring others. - -There are 2 methods to access and modify these options in `@extractus/article-extractor`. +```ts +import { getSanitizeHtmlOptions, setSanitizeHtmlOptions } from "jsr:@extractus/article-extractor"; -- `getSanitizeHtmlOptions()` -- `setSanitizeHtmlOptions(Object sanitizeHtmlOptions)` +// get current options (returns a clone) +const opts = getSanitizeHtmlOptions(); -Read [sanitize-html](https://github.com/apostrophecms/sanitize-html#default-options) docs for more info. +// merge new options +setSanitizeHtmlOptions({ + allowedAttributes: { + ...opts.allowedAttributes as Record, + code: ["class"], + }, +}); +``` --- -## Test +## Development ```bash git clone https://github.com/extractus/article-extractor.git cd article-extractor -pnpm i -pnpm test -``` -![article-extractor-test.png](https://i.imgur.com/TbRCUSS.png?110222) +# run tests +deno test --allow-all +# lint +deno lint -## Quick evaluation - -```bash -git clone https://github.com/extractus/article-extractor.git -cd article-extractor -pnpm i -pnpm eval {URL_TO_PARSE_ARTICLE} +# build npm package +deno run -A ./scripts/build_npm.ts ``` ## License The MIT License (MIT) -## Support the project - -If you find value from this open source project, you can support in the following ways: - -- Give it a star ⭐ -- Buy me a coffee: https://paypal.me/ndaidong đŸĩ -- Subscribe [Article Extractor service](https://rapidapi.com/pwshub-pwshub-default/api/article-extractor2) on RapidAPI 😉 - -Thank you. - --- diff --git a/deno.json b/deno.json new file mode 100644 index 0000000..d5692b7 --- /dev/null +++ b/deno.json @@ -0,0 +1,49 @@ +{ + "name": "@extractus/article-extractor", + "version": "9.0.0-rc0", + "description": "To extract main article from given URL", + "homepage": "https://github.com/extractus/article-extractor", + "repository": { + "type": "git", + "url": "git@github.com:extractus/article-extractor.git" + }, + "author": "@extractus", + "license": "MIT", + "exports": "./mod.ts", + "tasks": { + "test": "deno test --allow-all", + "lint": "deno lint", + "build": "deno run -A ./scripts/build_npm.ts" + }, + "imports": { + "@deno/dnt": "jsr:@deno/dnt@^0.43.2", + "@pwshub/bellajs": "jsr:@pwshub/bellajs@13.1.0", + "@std/assert": "jsr:@std/assert@1.0.12", + "@mozilla/readability": "npm:@mozilla/readability@^0.6.0", + "linkedom": "npm:linkedom@^0.18.13", + "sanitize-html": "npm:sanitize-html@^2.17.6", + "@types/sanitize-html": "npm:@types/sanitize-html@^2.16.1" + }, + "compilerOptions": { + "lib": ["dom", "dom.iterable", "dom.asynciterable", "deno.ns"] + }, + "lint": { + "include": ["src/", "mod.ts", "scripts/", "tests/"], + "exclude": ["npm/"], + "rules": { + "tags": ["recommended"], + "exclude": ["no-explicit-any"] + } + }, + "fmt": { + "include": ["src/", "mod.ts", "scripts/", "tests/"], + "exclude": ["npm/"] + }, + "test": { + "include": ["tests/"] + }, + "publish": { + "include": ["LICENSE", "README.md", "mod.ts", "src/"], + "exclude": ["tests/", "scripts/"] + } +} diff --git a/deno.lock b/deno.lock new file mode 100644 index 0000000..a211f92 --- /dev/null +++ b/deno.lock @@ -0,0 +1,268 @@ +{ + "version": "5", + "specifiers": { + "jsr:@david/code-block-writer@^13.0.3": "13.0.3", + "jsr:@deno/dnt@~0.43.2": "0.43.2", + "jsr:@pwshub/bellajs@13.1.0": "13.1.0", + "jsr:@std/assert@1.0.12": "1.0.12", + "jsr:@std/fmt@1": "1.0.10", + "jsr:@std/fs@1": "1.0.24", + "jsr:@std/internal@^1.0.14": "1.0.14", + "jsr:@std/internal@^1.0.6": "1.0.14", + "jsr:@std/path@1": "1.1.6", + "jsr:@std/path@^1.1.5": "1.1.6", + "jsr:@ts-morph/bootstrap@0.29": "0.29.0", + "jsr:@ts-morph/common@0.29": "0.29.0", + "npm:@mozilla/readability@0.6": "0.6.0", + "npm:@types/sanitize-html@^2.16.1": "2.16.1", + "npm:linkedom@~0.18.13": "0.18.13", + "npm:sanitize-html@^2.17.6": "2.17.6" + }, + "jsr": { + "@david/code-block-writer@13.0.3": { + "integrity": "f98c77d320f5957899a61bfb7a9bead7c6d83ad1515daee92dbacc861e13bb7f" + }, + "@deno/dnt@0.43.2": { + "integrity": "8068703a782f389d9fcb4ead32809ca666d3b088262a23f1d9016cbc9af5d37f", + "dependencies": [ + "jsr:@david/code-block-writer", + "jsr:@std/fmt", + "jsr:@std/fs", + "jsr:@std/path@1", + "jsr:@ts-morph/bootstrap" + ] + }, + "@pwshub/bellajs@13.1.0": { + "integrity": "3b2e3074c621edbab0514cd1737a1564765deefe55237f56663c54675cb5d315" + }, + "@std/assert@1.0.12": { + "integrity": "08009f0926dda9cbd8bef3a35d3b6a4b964b0ab5c3e140a4e0351fbf34af5b9a", + "dependencies": [ + "jsr:@std/internal@^1.0.6" + ] + }, + "@std/fmt@1.0.10": { + "integrity": "90dfba288802ac6de82fb31d0917eb9e4450b9925b954d5e51fc29ac07419db5" + }, + "@std/fs@1.0.24": { + "integrity": "f3061b45b81673a2bece689da041df32d174be064c89eb6397fb5718d3fb7877", + "dependencies": [ + "jsr:@std/internal@^1.0.14", + "jsr:@std/path@^1.1.5" + ] + }, + "@std/internal@1.0.14": { + "integrity": "291516b3d4c35024d6ffbc0a9df5bf4c64116e05b50012cf846710152d2ffdf7" + }, + "@std/path@1.1.6": { + "integrity": "c68485c2a4dfbb5ae3cc74fae4e8c4e5d874cf8a8ed12927917235c758b46cbe", + "dependencies": [ + "jsr:@std/internal@^1.0.14" + ] + }, + "@ts-morph/bootstrap@0.29.0": { + "integrity": "69f9b1b42b03423308a9f623f924174181ef20f6f2a8771799f216f561b9b7a6", + "dependencies": [ + "jsr:@ts-morph/common" + ] + }, + "@ts-morph/common@0.29.0": { + "integrity": "14e8f44c5e4297628cf756aed071787dc026c493e36c1cc29ce49dc95e493f8f", + "dependencies": [ + "jsr:@std/fs", + "jsr:@std/path@1" + ] + } + }, + "npm": { + "@mozilla/readability@0.6.0": { + "integrity": "sha512-juG5VWh4qAivzTAeMzvY9xs9HY5rAcr2E4I7tiSSCokRFi7XIZCAu92ZkSTsIj1OPceCifL3cpfteP3pDT9/QQ==" + }, + "@types/sanitize-html@2.16.1": { + "integrity": "sha512-n9wjs8bCOTyN/ynwD8s/nTcTreIHB1vf31vhLMGqUPNHaweKC4/fAl4Dj+hUlCTKYgm4P3k83fmiFfzkZ6sgMA==", + "dependencies": [ + "htmlparser2@10.1.0" + ] + }, + "boolbase@2.0.0": { + "integrity": "sha512-DkVaaQHymRhpYEYo9x1oo7Q7B0Y6KJUsjm3c9eTyFDby4MHLBTwZ6ZDWBel5zrYxj1WsZgC5oLpiz+93MluXeA==" + }, + "css-select@7.0.0": { + "integrity": "sha512-snmjEVXy+1LnwXdxhYvTMj1d9tOh4HxkA1YmoayVBeeyR2C14Pum7fcxJIm4SswYspVy866eYNwlH6xC3/VH5g==", + "dependencies": [ + "boolbase", + "css-what", + "domhandler@6.0.1", + "domutils@4.0.2", + "nth-check" + ] + }, + "css-what@8.0.0": { + "integrity": "sha512-DH0Bqq3DNp5tdOReuNyAA+Ev4Y2GS5FMbZpeTLP6C4CDi0h5nL0BmUPChXw3o/qbHLDWHl49sbNqQVY7bMSDdw==" + }, + "cssom@0.5.0": { + "integrity": "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw==" + }, + "dayjs@1.11.21": { + "integrity": "sha512-98IT+HOahAisibz/yjKbzuOBwYcjJ7BCLPzARyHiyEBmRz4fatF+KPJszEHXsGYjUG234aH/cOjW1wwTbKUZlA==" + }, + "deepmerge@4.3.1": { + "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==" + }, + "dom-serializer@2.0.0": { + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "dependencies": [ + "domelementtype@2.3.0", + "domhandler@5.0.3", + "entities@4.5.0" + ] + }, + "dom-serializer@3.1.1": { + "integrity": "sha512-4MEa38/QexBob6gFNwu+EGdWvhJ1OKuNwdYY3Y3NyeWDQfnGeDYQUDfIRzWu5B5gsv03so2Uxd28YC6zrsx3Lw==", + "dependencies": [ + "domelementtype@3.0.0", + "domhandler@6.0.1", + "entities@8.0.0" + ] + }, + "domelementtype@2.3.0": { + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==" + }, + "domelementtype@3.0.0": { + "integrity": "sha512-umCQid3jKbDmVjx8jGaW7uUykm4DEUeyV21hPxNMo2nV955DhUThwqyOIDtreepP31hl84X7G5U9ZfsWvIB3Pg==" + }, + "domhandler@5.0.3": { + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "dependencies": [ + "domelementtype@2.3.0" + ] + }, + "domhandler@6.0.1": { + "integrity": "sha512-gYzvtM72ZtxQO0T048kd6HWSbbGCNOUwcnfQ01cqIJ4X2IYKFFHZ5mKvrQETcFXxsRObZulDaKmy//R7TPtsBg==", + "dependencies": [ + "domelementtype@3.0.0" + ] + }, + "domutils@3.2.2": { + "integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==", + "dependencies": [ + "dom-serializer@2.0.0", + "domelementtype@2.3.0", + "domhandler@5.0.3" + ] + }, + "domutils@4.0.2": { + "integrity": "sha512-qI4JLRKnSzqFqr7hAlS5xQDusBCjKSEG4t4+7aNrIQMHBcsC2TGEhuyABJdYkgSewL57PNLYEiibY2iPKhKpaA==", + "dependencies": [ + "dom-serializer@3.1.1", + "domelementtype@3.0.0", + "domhandler@6.0.1" + ] + }, + "entities@4.5.0": { + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==" + }, + "entities@7.0.1": { + "integrity": "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==" + }, + "entities@8.0.0": { + "integrity": "sha512-zwfzJecQ/Uej6tusMqwAqU/6KL2XaB2VZ2Jg54Je6ahNBGNH6Ek6g3jjNCF0fG9EWQKGZNddNjU5F1ZQn/sBnA==" + }, + "escape-string-regexp@4.0.0": { + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==" + }, + "html-escaper@3.0.3": { + "integrity": "sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ==" + }, + "htmlparser2@10.1.0": { + "integrity": "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==", + "dependencies": [ + "domelementtype@2.3.0", + "domhandler@5.0.3", + "domutils@3.2.2", + "entities@7.0.1" + ] + }, + "htmlparser2@12.0.0": { + "integrity": "sha512-Tz7u1i95/g2x2jz81+x0FBVhBhY5aRTvD3tXXdFaljuNdzDLJ8UGNRrTcj2cgQvAg3iW/h77Fz15nLW0L0CrZw==", + "dependencies": [ + "domelementtype@3.0.0", + "domhandler@6.0.1", + "domutils@4.0.2", + "entities@8.0.0" + ] + }, + "is-plain-object@5.0.0": { + "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==" + }, + "launder@1.7.1": { + "integrity": "sha512-mU6WRz5EusL9ZZuiZ5SO4Y6C0P9PAUR9iwdb6bzj4KDihm28DiHFw+/yk9DBH4f+Pv1wuzQ4e2jV3oQ7mkIqvw==", + "dependencies": [ + "dayjs" + ] + }, + "linkedom@0.18.13": { + "integrity": "sha512-ES/o9qotMpzpN2MHs+Iq/JcVoOj8Fa5wiQYrTdFpvAnwXL0g66XHHUc9WUMk6nAlBtGsFQ24ne+SYnvnaQ2FSw==", + "dependencies": [ + "css-select", + "cssom", + "html-escaper", + "htmlparser2@10.1.0", + "uhyphen" + ] + }, + "nanoid@3.3.15": { + "integrity": "sha512-y7Wygv/7mEOvxTuEQDB8StXdMRBWf1kR/tlhAzBRUFkB2jfcLOAxO/SHmOO2zgz1pVgK29/kyupn059/bCHdjA==", + "bin": true + }, + "nth-check@3.0.1": { + "integrity": "sha512-GX0gsdbGVCgnRgbeGaubfjpBXyYRWOOCVeYh08bSQvDZqxz5ndXs1OTfAt/h36G1xvI94YIspsI0sVFqAV9+RQ==", + "dependencies": [ + "boolbase" + ] + }, + "parse-srcset@1.0.2": { + "integrity": "sha512-/2qh0lav6CmI15FzA3i/2Bzk2zCgQhGMkvhOhKNcBVQ1ldgpbfiNTVslmooUmWJcADi1f1kIeynbDRVzNlfR6Q==" + }, + "picocolors@1.1.1": { + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==" + }, + "postcss@8.5.16": { + "integrity": "sha512-vuwillviilfKZsg0VGj5R/YwwcHx4SLsIOI/7K6mQkWx+l5cUHTjj5g0AasTBcyXsbfTgrwsUNmVUb5xVwyPwg==", + "dependencies": [ + "nanoid", + "picocolors", + "source-map-js" + ] + }, + "sanitize-html@2.17.6": { + "integrity": "sha512-M4bo9tfv1yfhQZZKkc6dL07ALrGJtfvNOuhX3hU9AVPR/uPQ+nKOJBqTYc7LfMQblTW04mtSWDJWEyLvygJsLA==", + "dependencies": [ + "deepmerge", + "escape-string-regexp", + "htmlparser2@12.0.0", + "is-plain-object", + "launder", + "parse-srcset", + "postcss" + ] + }, + "source-map-js@1.2.1": { + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==" + }, + "uhyphen@0.2.0": { + "integrity": "sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA==" + } + }, + "workspace": { + "dependencies": [ + "jsr:@deno/dnt@~0.43.2", + "jsr:@pwshub/bellajs@13.1.0", + "jsr:@std/assert@1.0.12", + "npm:@mozilla/readability@0.6", + "npm:@types/sanitize-html@^2.16.1", + "npm:linkedom@~0.18.13", + "npm:sanitize-html@^2.17.6" + ] + } +} diff --git a/eslint.config.js b/eslint.config.js deleted file mode 100644 index 6701eb1..0000000 --- a/eslint.config.js +++ /dev/null @@ -1,127 +0,0 @@ -// eslint.config.js - -import eslintjs from '@eslint/js' -import globals from 'globals' - -export default [ - eslintjs.configs.recommended, - { - languageOptions: { - ecmaVersion: 'latest', - sourceType: 'module', - globals: { - ...globals.node, - ...globals.browser, - Intl: 'readonly', - }, - }, - ignores: [ - 'node_modules', - 'storage', - ], - rules: { - 'arrow-spacing': ['error', { 'before': true, 'after': true }], - 'block-spacing': ['error', 'always'], - 'brace-style': ['error', '1tbs', { 'allowSingleLine': true }], - 'camelcase': ['error', { - 'allow': ['^UNSAFE_'], - 'properties': 'never', - 'ignoreGlobals': true, - }], - 'comma-dangle': ['error', { - 'arrays': 'always-multiline', - 'objects': 'always-multiline', - 'imports': 'never', - 'exports': 'never', - 'functions': 'never', - }], - 'comma-spacing': ['error', { 'before': false, 'after': true }], - 'eol-last': 'error', - 'eqeqeq': ['error', 'always', { 'null': 'ignore' }], - 'func-call-spacing': ['error', 'never'], - 'indent': [ - 'error', - 2, - { - 'MemberExpression': 1, - 'FunctionDeclaration': { - 'body': 1, - 'parameters': 2, - }, - 'SwitchCase': 1, - 'ignoredNodes': ['TemplateLiteral > *'], - }, - ], - 'key-spacing': ['error', { 'beforeColon': false, 'afterColon': true }], - 'keyword-spacing': ['error', { 'before': true, 'after': true }], - 'lines-between-class-members': ['error', 'always', { 'exceptAfterSingleLine': true }], - 'max-len': [ - 'error', - { - 'code': 120, - 'ignoreTrailingComments': true, - 'ignoreComments': true, - 'ignoreUrls': true, - }, - ], - 'max-lines': [ - 'error', - { - 'max': 360, - 'skipBlankLines': true, - 'skipComments': false, - }, - ], - 'max-lines-per-function': [ - 'error', - { - 'max': 180, - 'skipBlankLines': true, - }, - ], - 'max-params': ['error', 3], - 'no-array-constructor': 'error', - 'no-mixed-spaces-and-tabs': 'error', - 'no-multi-spaces': 'error', - 'no-multi-str': 'error', - 'no-multiple-empty-lines': [ - 'error', - { - 'max': 1, - 'maxEOF': 0, - }, - ], - 'no-restricted-syntax': [ - 'error', - 'WithStatement', - 'BinaryExpression[operator=\'in\']', - ], - 'no-trailing-spaces': 'error', - 'no-use-before-define': [ - 'error', - { - 'functions': true, - 'classes': true, - 'variables': false, - }, - ], - 'no-var': 'warn', - 'object-curly-spacing': ['error', 'always'], - 'padded-blocks': [ - 'error', - { - 'blocks': 'never', - 'switches': 'never', - 'classes': 'never', - }, - ], - 'quotes': ['error', 'single'], - 'space-before-blocks': ['error', 'always'], - 'space-before-function-paren': ['error', 'always'], - 'space-infix-ops': 'error', - 'space-unary-ops': ['error', { 'words': true, 'nonwords': false }], - 'space-in-parens': ['error', 'never'], - 'semi': ['error', 'never'], - }, - }, -] diff --git a/eval.js b/eval.js deleted file mode 100644 index 993869b..0000000 --- a/eval.js +++ /dev/null @@ -1,53 +0,0 @@ -// eval.js - -import { execSync } from 'node:child_process' -import { readFileSync, writeFileSync, existsSync } from 'node:fs' - -import { slugify } from '@ndaidong/bellajs' - -import { isValid as isValidUrl } from './src/utils/linker.js' -import { extract, extractFromHtml } from './src/main.js' - -if (!existsSync('evaluation')) { - execSync('mkdir evaluation') -} - -const extractFromUrl = async (url) => { - try { - console.time('extraction') - const art = await extract(url) - console.log(art) - if (art) { - const slug = slugify(art.title) - writeFileSync(`evaluation/${slug}.html`, art.content, 'utf8') - } - console.timeEnd('extraction') - } catch (err) { - console.trace(err) - } -} - -const extractFromFile = async (fpath) => { - try { - const html = readFileSync(fpath, 'utf8') - const art = await extractFromHtml(html) - console.log(art) - if (art) { - const slug = slugify(art.title) - writeFileSync(`evaluation/${slug}.html`, art.content, 'utf8') - } - } catch (err) { - console.trace(err) - } -} - -const init = (argv) => { - if (argv.length === 3) { - const input = argv[2] - const isUrl = isValidUrl(input) - return isUrl ? extractFromUrl(input) : existsSync(input) ? extractFromFile(input) : false - } - return 'Nothing to do!' -} - -init(process.argv) diff --git a/index.d.ts b/index.d.ts deleted file mode 100644 index 27ff96a..0000000 --- a/index.d.ts +++ /dev/null @@ -1,124 +0,0 @@ -// Type definitions - -import { IOptions as SanitizeOptions } from 'sanitize-html' - -/** - * Transformation for per-site HTML pre/post processing. - */ -export interface Transformation { - /** URL regex patterns to match */ - patterns: Array - /** Function to pre-process raw HTML before extraction */ - pre?: (document: Document) => Document - /** Function to post-process extracted article content */ - post?: (document: Document) => Document -} - -/** - * Options for the article extraction process. - */ -export interface ParserOptions { - /** Words per minute for time-to-read estimation. Default: 300 */ - wordsPerMinute?: number - /** Max chars for generated description. Default: 210 */ - descriptionTruncateLen?: number - /** Min chars required for description. Default: 180 */ - descriptionLengthThreshold?: number - /** Min chars required for content. Default: 200 */ - contentLengthThreshold?: number -} - -/** - * Custom fetch function signature. - * - * @param url - URL to fetch - * @returns Promise resolving to a Response object - */ -export type Fetcher = (url: string) => Promise - -/** - * Extracted article data structure. - */ -export interface ArticleData { - /** Best resolved URL of the article */ - url?: string - /** Alternative URLs (canonical, shortlink, etc.) */ - links?: string[] - /** Article title */ - title?: string - /** Short description or excerpt */ - description?: string - /** Main image URL */ - image?: string - /** Site favicon URL */ - favicon?: string - /** Author name */ - author?: string - /** Extracted article HTML content */ - content?: string - /** Original publisher/source domain */ - source?: string - /** Publication date string */ - published?: string - /** Estimated time to read in seconds (0 = unknown) */ - ttr?: number - /** Page type (e.g. article) */ - type?: string -} - -/** - * Register one or more transformations for per-site HTML processing. - * - * @param transformations - Single transformation or array of transformations - * @returns Number of transformations successfully added - */ -export function addTransformations (transformations: Transformation | Array): number - -/** - * Remove transformations matching the given patterns. - * Calling without arguments removes all transformations. - * - * @param patterns - URL patterns to match for removal - * @returns Number of transformations removed - */ -export function removeTransformations (patterns?: Array): number - -/** - * Get a copy of the current sanitize-html options. - */ -export function getSanitizeHtmlOptions (): SanitizeOptions - -/** - * Update sanitize-html options by merging with the current ones. - * - * @param options - Partial sanitize options to merge - */ -export function setSanitizeHtmlOptions (options: SanitizeOptions): void - -/** - * Load and extract article data from a URL or HTML string. - * - * @param input - URL or HTML string to extract from - * @param parserOptions - Options for parsing - * @param fetcher - Custom fetch function (url) => Promise. Defaults to globalThis.fetch. - * @returns Extracted article data or null - */ -export function extract ( - input: string, - parserOptions?: ParserOptions, - fetcher?: Fetcher, -): Promise - -/** - * Extract article data from an HTML string directly. - * - * @param html - Raw HTML content - * @param url - Source URL for resolving relative links - * @param parserOptions - Options for parsing - * @returns Extracted article data or null - */ -export function extractFromHtml ( - html: string, - url?: string, - parserOptions?: ParserOptions, -): Promise diff --git a/mod.ts b/mod.ts new file mode 100644 index 0000000..52d0e7b --- /dev/null +++ b/mod.ts @@ -0,0 +1,14 @@ +export { + extract, + extractFromHtml, + addTransformations, + removeTransformations, + setSanitizeHtmlOptions, + getSanitizeHtmlOptions, +} from "./src/main.ts"; +export type { + ArticleData, + ParserOptions, + Fetcher, + Transformation, +} from "./src/main.ts"; diff --git a/package.json b/package.json deleted file mode 100644 index 31a7a29..0000000 --- a/package.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "version": "9.0.0-rc0", - "name": "@extractus/article-extractor", - "description": "To extract main article from given URL", - "homepage": "https://github.com/extractus/article-extractor", - "repository": { - "type": "git", - "url": "git@github.com:extractus/article-extractor.git" - }, - "author": "@extractus", - "main": "./src/main.js", - "type": "module", - "exports": { - ".": { - "types": "./index.d.ts", - "import": "./src/main.js", - "default": "./src/main.js" - } - }, - "types": "./index.d.ts", - "engines": { - "node": ">= 20" - }, - "scripts": { - "lint": "eslint .", - "lint:fix": "eslint --fix .", - "pretest": "npm run lint", - "test": "node --test --experimental-test-coverage", - "eval": "node eval", - "reset": "node reset" - }, - "dependencies": { - "@mozilla/readability": "^0.6.0", - "@pwshub/bellajs": "^13.1.0", - "linkedom": "^0.18.13", - "sanitize-html": "^2.17.6" - }, - "devDependencies": { - "@eslint/js": "^10.0.1", - "@types/sanitize-html": "^2.16.1", - "eslint": "^10.8.0", - "globals": "^17.9.0", - "https-proxy-agent": "^9.1.0", - "nock": "^14.0.17" - }, - "files": [ - "src", - "index.d.ts" - ], - "sideEffects": false, - "keywords": [ - "article", - "extractor", - "parser", - "readability", - "util" - ], - "license": "MIT" -} diff --git a/reset.js b/reset.js deleted file mode 100644 index 6afa5a0..0000000 --- a/reset.js +++ /dev/null @@ -1,34 +0,0 @@ -// reset.js - -import { - existsSync, - unlinkSync -} from 'node:fs' - -import { execSync } from 'node:child_process' - -const dirs = [ - 'evaluation', - 'docs', - '.nyc_output', - 'coverage', - 'node_modules', - '.nuxt', -] - -const files = [ - 'yarn.lock', - 'pnpm-lock.yaml', - 'package-lock.json', - 'coverage.lcov', -] - -dirs.forEach((d) => { - execSync(`rm -rf ${d}`) -}) - -files.forEach((f) => { - if (existsSync(f)) { - unlinkSync(f) - } -}) diff --git a/scripts/build_npm.ts b/scripts/build_npm.ts new file mode 100644 index 0000000..cdd4785 --- /dev/null +++ b/scripts/build_npm.ts @@ -0,0 +1,39 @@ +// scripts/build_npm.ts + +import { build, emptyDir } from "@deno/dnt"; +import pkg from "../deno.json" with { type: "json" }; + +const outputDir = "./npm"; + +await emptyDir(outputDir); + +await build({ + entryPoints: ["./mod.ts"], + outDir: outputDir, + shims: { + deno: true, + }, + test: false, + typeCheck: false, + tsconfig: { + compilerOptions: { + target: "es2022", + lib: ["es2022", "dom"], + }, + }, + package: { + name: pkg.name, + version: pkg.version, + description: pkg.description, + author: pkg.author, + repository: pkg.repository, + bugs: { + url: `${pkg.homepage}/issues`, + }, + license: pkg.license, + }, + postBuild() { + Deno.copyFileSync("LICENSE", "npm/LICENSE"); + Deno.copyFileSync("README.md", "npm/README.md"); + }, +}); diff --git a/src/config.js b/src/config.js deleted file mode 100644 index 1554625..0000000 --- a/src/config.js +++ /dev/null @@ -1,80 +0,0 @@ -// config.js - -import { clone } from '@pwshub/bellajs' - -/** - * Default sanitize-html options for cleaning extracted article content. - * Defines allowed HTML tags, attributes, and iframe domains. - * - * @type {SanitizeOptions} - */ -const sanitizeHtmlOptions = { - allowedTags: [ - 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', - 'u', 'b', 'i', 'em', 'strong', 'small', 'sup', 'sub', - 'div', 'span', 'p', 'article', 'blockquote', 'section', - 'details', 'summary', - 'pre', 'code', - 'ul', 'ol', 'li', 'dd', 'dl', - 'table', 'th', 'tr', 'td', 'thead', 'tbody', 'tfoot', - 'fieldset', 'legend', - 'figure', 'figcaption', 'img', 'picture', - 'video', 'audio', 'source', - 'iframe', - 'progress', - 'br', 'p', 'hr', - 'label', - 'abbr', - 'a', - 'svg', - ], - allowedAttributes: { - h1: ['id'], - h2: ['id'], - h3: ['id'], - h4: ['id'], - h5: ['id'], - h6: ['id'], - a: ['href', 'target', 'title'], - abbr: ['title'], - progress: ['value', 'max'], - img: ['src', 'srcset', 'alt', 'title'], - picture: ['media', 'srcset'], - video: ['controls', 'width', 'height', 'autoplay', 'muted', 'loop', 'src'], - audio: ['controls', 'width', 'height', 'autoplay', 'muted', 'loop', 'src'], - source: ['src', 'srcset', 'data-srcset', 'type', 'media', 'sizes'], - iframe: ['src', 'frameborder', 'height', 'width', 'scrolling', 'allow'], - svg: ['width', 'height'], // sanitize-html does not support svg fully yet - }, - allowedIframeDomains: [ - 'youtube.com', 'vimeo.com', 'odysee.com', - 'soundcloud.com', 'audius.co', - 'github.com', 'codepen.com', - 'twitter.com', 'facebook.com', 'instagram.com', - ], - disallowedTagsMode: 'discard', - allowVulnerableTags: false, - parseStyleAttributes: false, - enforceHtmlBoundary: false, -} - -/** - * Get a clone of the current sanitize-html options. - * - * @returns {SanitizeOptions} Cloned sanitize options - */ -export const getSanitizeHtmlOptions = () => { - return clone(sanitizeHtmlOptions) -} - -/** - * Update sanitize-html options by merging with the current ones. - * - * @param {SanitizeOptions} [opts={}] - Partial options to merge - * @returns {void} - */ -export const setSanitizeHtmlOptions = (opts = {}) => { - Object.keys(opts).forEach((key) => { - sanitizeHtmlOptions[key] = clone(opts[key]) - }) -} diff --git a/src/config.test.js b/src/config.test.js deleted file mode 100644 index cf3ba63..0000000 --- a/src/config.test.js +++ /dev/null @@ -1,37 +0,0 @@ -// config.test -import { describe, it } from 'node:test' -import assert from 'node:assert' - -import { - setSanitizeHtmlOptions, - getSanitizeHtmlOptions -} from './config.js' - -describe('check config methods', () => { - it('Testing setSanitizeHtmlOptions/getSanitizeHtmlOptions methods', () => { - setSanitizeHtmlOptions({ - allowedTags: ['div', 'span'], - allowedAttributes: { - a: ['href', 'title'], - }, - }) - - const actual = getSanitizeHtmlOptions() - const actualAllowedAttributes = actual.allowedAttributes - const expectedAllowedAttributes = { - a: ['href', 'title'], - } - - assert.deepEqual(actualAllowedAttributes, expectedAllowedAttributes) - - const actualAllowedTags = actual.allowedTags - const expectedAllowedTags = ['div', 'span'] - assert.deepEqual(actualAllowedTags, expectedAllowedTags) - - setSanitizeHtmlOptions({ - allowedTags: [], - }) - - assert.deepEqual(getSanitizeHtmlOptions().allowedTags, []) - }) -}) diff --git a/src/config.ts b/src/config.ts new file mode 100644 index 0000000..33aa9b4 --- /dev/null +++ b/src/config.ts @@ -0,0 +1,65 @@ +// config.ts + +import { clone } from "@pwshub/bellajs"; + +const sanitizeHtmlOptions: Record = { + allowedTags: [ + "h1", "h2", "h3", "h4", "h5", "h6", + "u", "b", "i", "em", "strong", "small", "sup", "sub", + "div", "span", "p", "article", "blockquote", "section", + "details", "summary", + "pre", "code", + "ul", "ol", "li", "dd", "dl", + "table", "th", "tr", "td", "thead", "tbody", "tfoot", + "fieldset", "legend", + "figure", "figcaption", "img", "picture", + "video", "audio", "source", + "iframe", + "progress", + "br", "p", "hr", + "label", + "abbr", + "a", + "svg", + ], + allowedAttributes: { + h1: ["id"], + h2: ["id"], + h3: ["id"], + h4: ["id"], + h5: ["id"], + h6: ["id"], + a: ["href", "target", "title"], + abbr: ["title"], + progress: ["value", "max"], + img: ["src", "srcset", "alt", "title"], + picture: ["media", "srcset"], + video: ["controls", "width", "height", "autoplay", "muted", "loop", "src"], + audio: ["controls", "width", "height", "autoplay", "muted", "loop", "src"], + source: ["src", "srcset", "data-srcset", "type", "media", "sizes"], + iframe: ["src", "frameborder", "height", "width", "scrolling", "allow"], + svg: ["width", "height"], + }, + allowedIframeDomains: [ + "youtube.com", "vimeo.com", "odysee.com", + "soundcloud.com", "audius.co", + "github.com", "codepen.com", + "twitter.com", "facebook.com", "instagram.com", + ], + disallowedTagsMode: "discard", + allowVulnerableTags: false, + parseStyleAttributes: false, + enforceHtmlBoundary: false, +}; + +/** Get a clone of the current sanitize-html options. */ +export const getSanitizeHtmlOptions = (): Record => { + return clone(sanitizeHtmlOptions); +}; + +/** Update sanitize-html options by merging with the current ones. */ +export const setSanitizeHtmlOptions = (opts: Record = {}): void => { + Object.keys(opts).forEach((key) => { + (sanitizeHtmlOptions as Record)[key] = clone(opts[key]); + }); +}; diff --git a/src/main.js b/src/main.js deleted file mode 100644 index 4142c84..0000000 --- a/src/main.js +++ /dev/null @@ -1,52 +0,0 @@ -// main.js - -import { - isString -} from '@pwshub/bellajs' - -import retrieve from './utils/retrieve.js' -import parseFromHtml from './utils/parseFromHtml.js' -import { getCharset } from './utils/html.js' -import { isValid as isValidUrl } from './utils/linker.js' - -/** - * Load and extract article data from a URL or HTML string. - * - * @param {string} input - URL or HTML string to extract from - * @param {object} [parserOptions={}] - Options for parsing - * @param {Function} [fetcher] - Custom fetch function (url) => Promise. Defaults to globalThis.fetch. - * @returns {Promise} Extracted article data or null - */ -export const extract = async (input, parserOptions = {}, fetcher = globalThis.fetch) => { - if (!isString(input)) { - throw new Error('Input must be a string') - } - - if (!isValidUrl(input)) { - return parseFromHtml(input, null, parserOptions) - } - const buffer = await retrieve(input, fetcher) - const text = buffer ? Buffer.from(buffer).toString().trim() : '' - if (!text) { - return null - } - const charset = getCharset(text) - const decoder = new TextDecoder(charset) - const html = decoder.decode(buffer) - return parseFromHtml(html, input, parserOptions) -} - -/** - * Extract article data from an HTML string directly. - * - * @param {string} html - Raw HTML content - * @param {string} [url] - Source URL for resolving relative links - * @param {ParserOptions} [parserOptions={}] - Options for parsing - * @returns {Promise} Extracted article data or null - */ -export const extractFromHtml = async (html, url, parserOptions = {}) => { - return parseFromHtml(html, url, parserOptions) -} - -export { addTransformations, removeTransformations } from './utils/transformation.js' -export { setSanitizeHtmlOptions, getSanitizeHtmlOptions } from './config.js' diff --git a/src/main.test.js b/src/main.test.js deleted file mode 100644 index a9251f0..0000000 --- a/src/main.test.js +++ /dev/null @@ -1,164 +0,0 @@ -// main.test - -import { describe, it } from 'node:test' -import assert from 'node:assert' - -import { readFileSync } from 'fs' - -import nock from 'nock' - -import { - extract, - getSanitizeHtmlOptions, - setSanitizeHtmlOptions, - addTransformations, - removeTransformations -} from './main.js' - -const env = process.env || {} -const PROXY_SERVER = env.PROXY_SERVER || '' - -const parseUrl = (url) => { - const re = new URL(url) - return { - baseUrl: `${re.protocol}//${re.host}`, - path: re.pathname, - } -} - -describe('check all exported methods', () => { - const fns = [ - extract, - getSanitizeHtmlOptions, - setSanitizeHtmlOptions, - addTransformations, - removeTransformations, - ] - - fns.forEach((fn) => { - it(` check ${fn.name}`, () => { - assert.ok(fn) - }) - }) -}) - -describe('test extract(bad url)', () => { - const badSamples = [ - '', - { k: 9 }, - [1, 3, 4], - 301932, - 'htt:/abc.com/failed-none-sense', - 'fpt://abc.com/failed-none-sense', - 'ttp://badcom/146753785', - 'https://674458092126388225', - 'https://soundcloud^(*%%$%^$$%$$*&(&)())', - ] - - badSamples.forEach((url) => { - it(`testing extract bad url "${url}"`, async () => { - try { - await extract(url) - } catch (err) { - assert.ok(err) - } - }) - }) -}) - -describe('test extract(regular article url)', () => { - const expDesc = [ - 'Navigation here Few can name a rational peach that isn\'t a conscientious goldfish!', - 'One cannot separate snakes from plucky pomegranates?', - 'Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs.', - ].join(' ') - const cases = [ - { - input: { - url: 'https://somewhere.com/path/to/no/article', - html: readFileSync('./test-data/html-no-article.html', 'utf8'), - }, - validate: (result) => { - assert.equal(result, null) - }, - }, - { - input: { - url: 'https://somewhere.com/path/to/no/content', - html: '', - }, - validate: (result) => { - assert.equal(result, null) - }, - }, - { - input: { - url: 'https://somewhere.com/path/to/article', - html: readFileSync('./test-data/regular-article.html', 'utf8'), - }, - validate: (result) => { - assert.ok(result) - assert.equal(result.title, 'Article title here') - assert.equal(result.description, expDesc) - }, - }, - ] - cases.forEach(({ input, validate }) => { - const { url, html, statusCode = 200 } = input - const { baseUrl, path } = parseUrl(url) - const scope = nock(baseUrl) - scope.get(path) - .reply(statusCode, html, { - 'Content-Type': 'text/html', - }) - it(`check extract("${url}")`, async () => { - const result = await extract(url) - validate(result) - }) - }) - - it('check extract(html string)', async () => { - const html = readFileSync('./test-data/regular-article.html', 'utf8') - const result = await extract(html) - assert.ok(result) - assert.equal(result.title, 'Article title here') - assert.equal(result.description, expDesc) - }) -}) - -describe('test extract with modified sanitize-html options', () => { - const currentSanitizeOptions = getSanitizeHtmlOptions() - - setSanitizeHtmlOptions({ - ...currentSanitizeOptions, - allowedAttributes: { - ...currentSanitizeOptions.allowedAttributes, - code: ['class'], - div: ['class'], - }, - allowedClasses: { - code: ['language-*', 'lang-*'], - }, - }) - - it('check if output contain class attribute', async () => { - const html = readFileSync('./test-data/article-with-classes-attributes.html', 'utf8') - const result = await extract(html) - assert.ok(result.content.includes('code class="lang-js"')) - }) -}) - -if (PROXY_SERVER !== '') { - describe('test extract live article API via custom fetcher', () => { - it('check if extract method works with custom fetcher', async () => { - const { HttpsProxyAgent } = await import('https-proxy-agent') - const url = 'https://www.cnbc.com/2022/09/21/what-another-major-rate-hike-by-the-federal-reserve-means-to-you.html' - const myFetcher = (fetchUrl) => fetch(fetchUrl, { - agent: new HttpsProxyAgent(PROXY_SERVER), - }) - const result = await extract(url, {}, myFetcher) - assert.ok(result.title.includes('Federal Reserve')) - assert.equal(result.source, 'cnbc.com') - }, 10000) - }) -} diff --git a/src/main.ts b/src/main.ts new file mode 100644 index 0000000..e1c0236 --- /dev/null +++ b/src/main.ts @@ -0,0 +1,112 @@ +// main.ts + +import { isString } from "@pwshub/bellajs"; + +import retrieve from "./utils/retrieve.ts"; +import parseFromHtml from "./utils/parseFromHtml.ts"; +import { getCharset } from "./utils/html.ts"; +import { isValid as isValidUrl } from "./utils/linker.ts"; + +/** Custom fetch function signature. */ +export type Fetcher = (url: string) => Promise; + +/** Per-site HTML pre/post processing transformation. */ +export interface Transformation { + /** URL regex patterns to match against article links. */ + patterns: RegExp[]; + /** Function to pre-process raw HTML before extraction. */ + pre?: (document: Document) => Document; + /** Function to post-process extracted article content. */ + post?: (document: Document) => Document; +} + +/** Options for the article extraction process. */ +export interface ParserOptions { + /** Words per minute for time-to-read estimation. Default: 300 */ + wordsPerMinute?: number; + /** Max characters for generated description. Default: 210 */ + descriptionTruncateLen?: number; + /** Min characters required to keep meta description. Default: 180 */ + descriptionLengthThreshold?: number; + /** Min characters required for article content. Default: 200 */ + contentLengthThreshold?: number; +} + +/** Extracted article data structure. */ +export interface ArticleData { + /** Best resolved URL of the article. */ + url?: string; + /** Alternative URLs discovered (canonical, shortlink, amphtml). */ + links?: string[]; + /** Article title. */ + title?: string; + /** Short description or excerpt. */ + description?: string; + /** Main image URL. */ + image?: string; + /** Site favicon URL. */ + favicon?: string; + /** Author name. */ + author?: string; + /** Extracted article HTML content. */ + content?: string; + /** Original publisher/source domain. */ + source?: string; + /** Publication date string. */ + published?: string; + /** Estimated time to read in seconds. */ + ttr?: number; + /** Page type (e.g. "article"). */ + type?: string; +} + +/** + * Load and extract article data from a URL or HTML string. + * + * @param input - URL or HTML string to extract from + * @param parserOptions - Options for parsing + * @param fetcher - Custom fetch function. Defaults to globalThis.fetch. + * @returns Extracted article data or null + */ +export const extract = async ( + input: string, + parserOptions: ParserOptions = {}, + fetcher: Fetcher = globalThis.fetch, +): Promise => { + if (!isString(input)) { + throw new Error("Input must be a string"); + } + + if (!isValidUrl(input)) { + return parseFromHtml(input, "", parserOptions); + } + const buffer = await retrieve(input, fetcher); + const text = buffer ? new TextDecoder().decode(buffer).trim() : ""; + if (!text) { + return null; + } + const charset = getCharset(text); + const decoder = new TextDecoder(charset); + const html = decoder.decode(buffer); + return parseFromHtml(html, input, parserOptions); +}; + +/** + * Extract article data from an HTML string directly. + * + * @param html - Raw HTML content + * @param url - Source URL for resolving relative links + * @param parserOptions - Options for parsing + * @returns Extracted article data or null + */ +// deno-lint-ignore require-await +export const extractFromHtml = async ( + html: string, + url?: string, + parserOptions: ParserOptions = {}, +): Promise => { + return parseFromHtml(html, url || "", parserOptions); +}; + +export { addTransformations, removeTransformations } from "./utils/transformation.ts"; +export { setSanitizeHtmlOptions, getSanitizeHtmlOptions } from "./config.ts"; diff --git a/src/utils/extractLdSchema.js b/src/utils/extractLdSchema.js deleted file mode 100644 index d4cc82d..0000000 --- a/src/utils/extractLdSchema.js +++ /dev/null @@ -1,107 +0,0 @@ -// utils -> extractLdSchema.js - -import { isArray, isObject, isString } from '@pwshub/bellajs' - -/** - * Allowed JSON-LD schema types that indicate an article or webpage. - * - * @type {string[]} - */ -const typeSchemas = [ - 'aboutpage', - 'checkoutpage', - 'collectionpage', - 'contactpage', - 'faqpage', - 'itempage', - 'medicalwebpage', - 'profilepage', - 'qapage', - 'realestatelisting', - 'searchresultspage', - 'webpage', - 'website', - 'article', - 'advertisercontentarticle', - 'newsarticle', - 'analysisnewsarticle', - 'askpublicnewsarticle', - 'backgroundnewsarticle', - 'opinionnewsarticle', - 'reportagenewsarticle', - 'reviewnewsarticle', - 'report', - 'satiricalarticle', - 'scholarlyarticle', - 'medicalscholarlyarticle', -] - -/** - * Mapping from entry keys to JSON-LD attribute names. - * - * @type {Object} - */ -const attributeLists = { - description: 'description', - image: 'image', - author: 'author', - published: 'datePublished', - type: '@type', -} - -/** - * Safely parse a JSON string, returning an empty object on failure. - * - * @param {string} text - JSON string to parse - * @returns {Object} Parsed object or empty object - */ -const parseJson = (text) => { - try { - return JSON.parse(text) - } catch { - return {} - } -} - -/** - * Check if the given JSON-LD object has an allowed schema type. - * - * @param {Object} ldJson - Parsed JSON-LD object - * @returns {boolean} True if type is in the allowed list - */ -const isAllowedLdJsonType = (ldJson) => { - const rootLdJsonType = ldJson['@type'] || '' - const arr = isArray(rootLdJsonType) ? rootLdJsonType : [rootLdJsonType] - const ldJsonTypes = arr.filter(x => !!x) - return ldJsonTypes.length > 0 && ldJsonTypes.some(x => typeSchemas.includes(x.toLowerCase())) -} - -/** - * Parses JSON-LD data from a document and populates an entry object. - * Only populates if the original entry object is empty or undefined. - * - * @param {Document} document - The HTML Document - * @param {Object} entry - The entry object to merge/populate with JSON-LD. - * @returns {Object} The entry object after being merged/populated with data. - */ -export default (document, entry) => { - const ldSchemas = document.querySelectorAll('script[type="application/ld+json"]') - ldSchemas.forEach(ldSchema => { - const ldJson = parseJson(ldSchema.textContent.replace(/[\n\r\t]/g, '')) - if (ldJson && isAllowedLdJsonType(ldJson)) { - for (const [key, attr] of Object.entries(attributeLists)) { - if (entry[key] || !ldJson[attr]) { - continue - } - - const keyValue = ldJson[attr] - const val = isArray(keyValue) ? keyValue[0] : isObject(keyValue) ? keyValue?.name || '' : keyValue - if (isString(val) && val !== '') { - entry[key] = val.trim() - } - } - } - }) - - return entry -} diff --git a/src/utils/extractLdSchema.ts b/src/utils/extractLdSchema.ts new file mode 100644 index 0000000..7d6ed1b --- /dev/null +++ b/src/utils/extractLdSchema.ts @@ -0,0 +1,77 @@ +// utils -> extractLdSchema.ts + +import { isArray, isObject, isString } from "@pwshub/bellajs"; + +const typeSchemas = [ + "aboutpage", + "checkoutpage", + "collectionpage", + "contactpage", + "faqpage", + "itempage", + "medicalwebpage", + "profilepage", + "qapage", + "realestatelisting", + "searchresultspage", + "webpage", + "website", + "article", + "advertisercontentarticle", + "newsarticle", + "analysisnewsarticle", + "askpublicnewsarticle", + "backgroundnewsarticle", + "opinionnewsarticle", + "reportagenewsarticle", + "reviewnewsarticle", + "report", + "satiricalarticle", + "scholarlyarticle", + "medicalscholarlyarticle", +]; + +const attributeLists: Record = { + description: "description", + image: "image", + author: "author", + published: "datePublished", + type: "@type", +}; + +const parseJson = (text: string): Record => { + try { + return JSON.parse(text); + } catch { + return {}; + } +}; + +const isAllowedLdJsonType = (ldJson: Record): boolean => { + const rootLdJsonType = ldJson["@type"] || ""; + const arr = isArray(rootLdJsonType) ? rootLdJsonType as string[] : [rootLdJsonType as string]; + const ldJsonTypes = arr.filter((x) => !!x); + return ldJsonTypes.length > 0 && ldJsonTypes.some((x) => typeSchemas.includes(x.toLowerCase())); +}; + +export default (document: Document, entry: Record): Record => { + const ldSchemas = document.querySelectorAll('script[type="application/ld+json"]'); + ldSchemas.forEach((ldSchema: Element) => { + const ldJson = parseJson(ldSchema.textContent.replace(/[\n\r\t]/g, "")); + if (ldJson && isAllowedLdJsonType(ldJson)) { + for (const [key, attr] of Object.entries(attributeLists)) { + if (entry[key] || !ldJson[attr]) { + continue; + } + + const keyValue = ldJson[attr]; + const val = isArray(keyValue) ? (keyValue as unknown[])[0] : isObject(keyValue) ? (keyValue as Record)?.name || "" : keyValue; + if (isString(val) && val !== "") { + entry[key] = (val as string).trim(); + } + } + } + }); + + return entry; +}; diff --git a/src/utils/extractMetaData.js b/src/utils/extractMetaData.js deleted file mode 100644 index 315e7fe..0000000 --- a/src/utils/extractMetaData.js +++ /dev/null @@ -1,154 +0,0 @@ -// utils -> extractMetaData - -import { DOMParser } from 'linkedom' -import extractLdSchema from './extractLdSchema.js' -import findDate from './findDate.js' - -/** - * @param {Element} node - * @param {Object} attributeLists - * @returns {?{key: string, content: string}} - */ -function getMetaContentByNameOrProperty (node, attributeLists) { - const content = node.getAttribute('content') - if (!content) return null - - const property = node - .getAttribute('property')?.toLowerCase() ?? - node.getAttribute('itemprop')?.toLowerCase() - - const name = node.getAttribute('name')?.toLowerCase() - - for (const [key, attrs] of Object.entries(attributeLists)) { - if (attrs.includes(property) || attrs.includes(name)) { - return { key, content } - } - } - - return null -} - -/** - * @param html {string} - * @returns {{image: string, author: string, amphtml: string, description: string, canonical: string, source: string, published: string, title: string, url: string, shortlink: string, favicon: string, type: string}} - */ -export default (html) => { - const entry = { - url: '', - shortlink: '', - amphtml: '', - canonical: '', - title: '', - description: '', - image: '', - author: '', - source: '', - published: '', - favicon: '', - type: '', - } - - const sourceAttrs = [ - 'application-name', - 'og:site_name', - 'twitter:site', - 'dc.title', - ] - const urlAttrs = [ - 'og:url', - 'twitter:url', - 'parsely-link', - ] - const titleAttrs = [ - 'title', - 'og:title', - 'twitter:title', - 'parsely-title', - ] - const descriptionAttrs = [ - 'description', - 'og:description', - 'twitter:description', - 'parsely-description', - ] - const imageAttrs = [ - 'image', - 'og:image', - 'og:image:url', - 'og:image:secure_url', - 'twitter:image', - 'twitter:image:src', - 'parsely-image-url', - ] - const authorAttrs = [ - 'author', - 'creator', - 'og:creator', - 'article:author', - 'twitter:creator', - 'dc.creator', - 'parsely-author', - ] - const publishedTimeAttrs = [ - 'article:published_time', - 'article:modified_time', - 'og:updated_time', - 'dc.date', - 'dc.date.issued', - 'dc.date.created', - 'dc:created', - 'dcterms.date', - 'datepublished', - 'datemodified', - 'updated_time', - 'modified_time', - 'published_time', - 'release_date', - 'date', - 'parsely-pub-date', - ] - const typeAttrs = [ - 'og:type', - ] - - const attributeLists = { - source: sourceAttrs, - url: urlAttrs, - title: titleAttrs, - description: descriptionAttrs, - image: imageAttrs, - author: authorAttrs, - published: publishedTimeAttrs, - type: typeAttrs, - } - - const doc = new DOMParser().parseFromString(html, 'text/html') - entry.title = doc.querySelector('head > title')?.innerText - - Array.from(doc.getElementsByTagName('link')).forEach(node => { - const rel = node.getAttribute('rel') - const href = node.getAttribute('href') - if (rel && href) { - entry[rel] = href - if (rel === 'icon' || rel === 'shortcut icon') { - entry.favicon = href - } - } - }) - - Array.from(doc.getElementsByTagName('meta')).forEach(node => { - const result = getMetaContentByNameOrProperty(node, attributeLists) - const val = result?.content || '' - if (val !== '') { - entry[result.key] = val - } - }) - - const metadata = extractLdSchema(doc, entry) - - if (!metadata.published) { - metadata.published = findDate(doc) || '' - } - - return metadata -} diff --git a/src/utils/extractMetaData.test.js b/src/utils/extractMetaData.test.js deleted file mode 100644 index dd0e681..0000000 --- a/src/utils/extractMetaData.test.js +++ /dev/null @@ -1,78 +0,0 @@ -// extractMetaData.test -import { describe, it } from 'node:test' -import assert from 'node:assert' - -import { readFileSync } from 'node:fs' - -import { isObject, hasProperty } from '@pwshub/bellajs' - -import extractMetaData from './extractMetaData.js' - -const keys = 'url shortlink amphtml canonical title description image author source published favicon type'.split(' ') - -function isDateString (date) { - if (typeof date !== 'string') return false - const d = new Date(date) - return !isNaN(d.getTime()) -} - -describe('test extractMetaData', () => { - it('test extractMetaData(good content)', async () => { - const html = readFileSync('./test-data/regular-article.html', 'utf8') - const result = extractMetaData(html) - assert.ok(isObject(result)) - keys.forEach((k) => { - assert.ok(hasProperty(result, k)) - }) - }) - - it('test extractMetaData(json ld schema content)', async () => { - const html = readFileSync('./test-data/regular-article-json-ld.html', 'utf8') - const result = extractMetaData(html) - assert.ok(isObject(result)) - keys.forEach((k) => { - assert.ok(hasProperty(result, k)) - }) - }) - - it('test extractMetaData(json ld schema fills empty meta fields)', async () => { - const ldJson = { - '@context': 'https://schema.org', - '@type': 'NewsArticle', - author: { name: 'JSON Author' }, - datePublished: '2026-01-02T03:04:05Z', - description: 'JSON description', - image: 'https://example.com/json.jpg', - } - const body = '

' + 'Article text. '.repeat(50) + '

' - const html = ` -Fixture article - -

Fixture article

${body}
` - const result = extractMetaData(html) - assert.strictEqual(result.author, 'JSON Author') - assert.strictEqual(result.published, '2026-01-02T03:04:05Z') - assert.strictEqual(result.description, 'JSON description') - assert.strictEqual(result.image, 'https://example.com/json.jpg') - }) - - it('test extractMetaData(find date)', async () => { - const html1 = readFileSync('./test-data/regular-article-date-time.html', 'utf8') - const html2 = readFileSync('./test-data/regular-article-date-itemprop.html', 'utf8') - const html3 = readFileSync('./test-data/regular-article-date-span.html', 'utf8') - const result1 = extractMetaData(html1) - const result2 = extractMetaData(html2) - const result3 = extractMetaData(html3) - assert.ok(isObject(result1)) - assert.ok(isObject(result2)) - assert.ok(isObject(result3)) - keys.forEach((k) => { - assert.ok(hasProperty(result1, k)) - assert.ok(hasProperty(result3, k)) - assert.ok(hasProperty(result3, k)) - }) - assert.ok(isDateString(result1.published)) - assert.ok(isDateString(result2.published)) - assert.ok(isDateString(result3.published)) - }) -}) diff --git a/src/utils/extractMetaData.ts b/src/utils/extractMetaData.ts new file mode 100644 index 0000000..8178c3c --- /dev/null +++ b/src/utils/extractMetaData.ts @@ -0,0 +1,148 @@ +// utils -> extractMetaData.ts + +import { DOMParser } from "linkedom"; +import extractLdSchema from "./extractLdSchema.ts"; +import findDate from "./findDate.ts"; + +function getMetaContentByNameOrProperty( + node: Element, + attributeLists: Record, +): { key: string; content: string } | null { + const content = node.getAttribute("content"); + if (!content) return null; + + const property = node + .getAttribute("property")?.toLowerCase() ?? + node.getAttribute("itemprop")?.toLowerCase(); + + const name = node.getAttribute("name")?.toLowerCase(); + + for (const [key, attrs] of Object.entries(attributeLists)) { + if (attrs.includes(property || "") || attrs.includes(name || "")) { + return { key, content }; + } + } + + return null; +} + +export default (html: string): Record => { + const entry: Record = { + url: "", + shortlink: "", + amphtml: "", + canonical: "", + title: "", + description: "", + image: "", + author: "", + source: "", + published: "", + favicon: "", + type: "", + }; + + const sourceAttrs = [ + "application-name", + "og:site_name", + "twitter:site", + "dc.title", + ]; + const urlAttrs = [ + "og:url", + "twitter:url", + "parsely-link", + ]; + const titleAttrs = [ + "title", + "og:title", + "twitter:title", + "parsely-title", + ]; + const descriptionAttrs = [ + "description", + "og:description", + "twitter:description", + "parsely-description", + ]; + const imageAttrs = [ + "image", + "og:image", + "og:image:url", + "og:image:secure_url", + "twitter:image", + "twitter:image:src", + "parsely-image-url", + ]; + const authorAttrs = [ + "author", + "creator", + "og:creator", + "article:author", + "twitter:creator", + "dc.creator", + "parsely-author", + ]; + const publishedTimeAttrs = [ + "article:published_time", + "article:modified_time", + "og:updated_time", + "dc.date", + "dc.date.issued", + "dc.date.created", + "dc:created", + "dcterms.date", + "datepublished", + "datemodified", + "updated_time", + "modified_time", + "published_time", + "release_date", + "date", + "parsely-pub-date", + ]; + const typeAttrs = [ + "og:type", + ]; + + const attributeLists: Record = { + source: sourceAttrs, + url: urlAttrs, + title: titleAttrs, + description: descriptionAttrs, + image: imageAttrs, + author: authorAttrs, + published: publishedTimeAttrs, + type: typeAttrs, + }; + + const doc = new DOMParser().parseFromString(html, "text/html"); + entry.title = doc.querySelector("head > title")?.innerText || ""; + + Array.from(doc.getElementsByTagName("link")).forEach((node) => { + const rel = node.getAttribute("rel"); + const href = node.getAttribute("href"); + if (rel && href) { + entry[rel] = href; + if (rel === "icon" || rel === "shortcut icon") { + entry.favicon = href; + } + } + }); + + Array.from(doc.getElementsByTagName("meta")).forEach((node) => { + const result = getMetaContentByNameOrProperty(node, attributeLists); + const val = result?.content || ""; + if (val !== "" && result) { + entry[result.key] = val; + } + }); + + const metadata = extractLdSchema(doc as any, entry); + + if (!metadata.published) { + metadata.published = findDate(doc as any) || ""; + } + + return metadata; +}; diff --git a/src/utils/extractWithReadability.js b/src/utils/extractWithReadability.js deleted file mode 100644 index 56833d9..0000000 --- a/src/utils/extractWithReadability.js +++ /dev/null @@ -1,42 +0,0 @@ -// utils/extractWithReadability - -import { Readability } from '@mozilla/readability' -import { DOMParser } from 'linkedom' -import { isString } from '@pwshub/bellajs' - -/** - * Extract main article content from HTML using Mozilla Readability. - * - * @param {string} html - Raw HTML content - * @param {string} [url=''] - Source URL for resolving relative paths - * @returns {string|null} Extracted article HTML or null - */ -export default (html, url = '') => { - if (!isString(html)) { - return null - } - const doc = new DOMParser().parseFromString(html, 'text/html') - const base = doc.createElement('base') - base.setAttribute('href', url) - doc.head.appendChild(base) - const reader = new Readability(doc, { - keepClasses: true, - }) - const result = reader.parse() ?? {} - return result.textContent ? result.content : null -} - -/** - * Extract article title from HTML using Mozilla Readability. - * - * @param {string} html - Raw HTML content - * @returns {string|null} Extracted title or null - */ -export function extractTitleWithReadability (html) { - if (!isString(html)) { - return null - } - const doc = new DOMParser().parseFromString(html, 'text/html') - const reader = new Readability(doc) - return reader._getArticleTitle() || null -} diff --git a/src/utils/extractWithReadability.test.js b/src/utils/extractWithReadability.test.js deleted file mode 100644 index 80f4bde..0000000 --- a/src/utils/extractWithReadability.test.js +++ /dev/null @@ -1,43 +0,0 @@ -// extractWithReadability.test - -import { describe, it } from 'node:test' -import assert from 'node:assert' - -import { readFileSync } from 'node:fs' - -import { isString } from '@pwshub/bellajs' - -import extractWithReadability, { extractTitleWithReadability } from './extractWithReadability.js' - -describe('test extractWithReadability()', () => { - it('extract from good html content', async () => { - const html = readFileSync('./test-data/regular-article.html', 'utf8') - const result = extractWithReadability(html, 'https://foo.bar') - assert.ok(isString(result)) - assert.ok(result.length > 200) - assert.ok(result.includes('')) - }) - - it('extract from bad html content', async () => { - assert.equal(extractWithReadability(null), null) - assert.equal(extractWithReadability({}), null) - assert.equal(extractWithReadability('
'), null) - }) - - it('extract title only', async () => { - const html = readFileSync('./test-data/regular-article.html', 'utf8') - const result = extractTitleWithReadability(html) - assert.equal(result, 'Article title here - ArticleParser') - }) - - it('extract title from page without title', async () => { - const html = readFileSync('./test-data/html-no-title.html', 'utf8') - const result = extractTitleWithReadability(html) - assert.equal(result, null) - }) - - it('extract title from non-string', async () => { - const result = extractTitleWithReadability({}) - assert.equal(result, null) - }) -}) diff --git a/src/utils/extractWithReadability.ts b/src/utils/extractWithReadability.ts new file mode 100644 index 0000000..e206784 --- /dev/null +++ b/src/utils/extractWithReadability.ts @@ -0,0 +1,29 @@ +// utils -> extractWithReadability.ts + +import { Readability } from "@mozilla/readability"; +import { DOMParser } from "linkedom"; +import { isString } from "@pwshub/bellajs"; + +export default (html: string, url = ""): string | null => { + if (!isString(html)) { + return null; + } + const doc = new DOMParser().parseFromString(html, "text/html"); + const base = doc.createElement("base", {}); + base.setAttribute("href", url); + doc.head.appendChild(base); + const reader = new Readability(doc as unknown as Document, { + keepClasses: true, + }); + const result: Record = reader.parse() ?? {}; + return result.textContent ? result.content as string : null; +}; + +export function extractTitleWithReadability(html: string): string | null { + if (!isString(html)) { + return null; + } + const doc = new DOMParser().parseFromString(html, "text/html"); + const reader = new Readability(doc as unknown as Document); + return (reader as any)._getArticleTitle() || null; +} diff --git a/src/utils/findDate.js b/src/utils/findDate.js deleted file mode 100644 index 242eaa6..0000000 --- a/src/utils/findDate.js +++ /dev/null @@ -1,59 +0,0 @@ - -/** - * Convert date format to YYYY-MM-DD - * - * @param {string} dateString - * @returns {string} YYYY-MM-DD - */ -function convertDateFormat (dateString) { - const parts = dateString.split('/') - if (parts.length !== 3) return dateString - - let year, month, day - - if (parts[0].length === 4 || parseInt(parts[0]) > 31) { - [year, month, day] = parts - } else if (parseInt(parts[0]) > 12) { - [day, month, year] = parts - } else { - [month, day, year] = parts - } - - year = year.length === 2 ? '20' + year : year - return `${year}-${month.padStart(2, '0')}-${day.padStart(2, '0')}T00:00:00` -} - -/** - * Look for the publication date in the body of the content. - * - * @param {Document} document - The HTML Document - * @returns {string} The date string - */ -export default function (doc) { - const datePatterns = [ - /\d{4}-\d{2}-\d{2}/, - /\d{1,2}\/\d{1,2}\/\d{2,4}/, - ] - - const findDate = (element) => { - for (const pattern of datePatterns) { - const match = element.textContent.match(pattern) - if (match) return convertDateFormat(match[0]) - } - return '' - } - - const priorityElements = doc.querySelectorAll('time, [datetime], [itemprop~=datePublished], [itemprop~=dateCreated]') - for (const el of priorityElements) { - const date = el.getAttribute('datetime') || el.getAttribute('content') || findDate(el) - if (date) return date - } - - const secondaryElements = doc.querySelectorAll('p, span, div') - for (const el of secondaryElements) { - const date = findDate(el) - if (date) return date - } - - return '' -} diff --git a/src/utils/findDate.ts b/src/utils/findDate.ts new file mode 100644 index 0000000..d6bcca8 --- /dev/null +++ b/src/utils/findDate.ts @@ -0,0 +1,48 @@ +// utils -> findDate.ts + +function convertDateFormat(dateString: string): string { + const parts = dateString.split("/"); + if (parts.length !== 3) return dateString; + + let year: string, month: string, day: string; + + if (parts[0].length === 4 || parseInt(parts[0]) > 31) { + [year, month, day] = parts; + } else if (parseInt(parts[0]) > 12) { + [day, month, year] = parts; + } else { + [month, day, year] = parts; + } + + year = year.length === 2 ? "20" + year : year; + return `${year}-${month.padStart(2, "0")}-${day.padStart(2, "0")}T00:00:00`; +} + +export default function findDate(doc: Document): string { + const datePatterns = [ + /\d{4}-\d{2}-\d{2}/, + /\d{1,2}\/\d{1,2}\/\d{2,4}/, + ]; + + const findDateInElement = (element: Element): string => { + for (const pattern of datePatterns) { + const match = element.textContent?.match(pattern); + if (match) return convertDateFormat(match[0]); + } + return ""; + }; + + const priorityElements = doc.querySelectorAll("time, [datetime], [itemprop~=datePublished], [itemprop~=dateCreated]"); + for (const el of Array.from(priorityElements)) { + const date = el.getAttribute("datetime") || el.getAttribute("content") || findDateInElement(el); + if (date) return date; + } + + const secondaryElements = doc.querySelectorAll("p, span, div"); + for (const el of Array.from(secondaryElements)) { + const date = findDateInElement(el); + if (date) return date; + } + + return ""; +} diff --git a/src/utils/html.js b/src/utils/html.js deleted file mode 100644 index 33bbaa6..0000000 --- a/src/utils/html.js +++ /dev/null @@ -1,100 +0,0 @@ -// utils -> html - -import { DOMParser } from 'linkedom' -import sanitize from 'sanitize-html' -import { pipe } from '@pwshub/bellajs' - -import { getSanitizeHtmlOptions } from '../config.js' - -/** - * Lightweight HTML sanitization that fixes structural issues - * without stripping any tags or attributes. - * - * @param {string} html - Raw HTML input - * @returns {string} Sanitized HTML (all tags/attributes preserved) - */ -export const purify = (html) => { - return sanitize(html, { - allowedTags: false, - allowedAttributes: false, - allowVulnerableTags: true, - }) -} - -/** - * Regex matching strings that consist entirely of whitespace characters. - * - * @type {RegExp} - */ -const WS_REGEXP = /^[\s\f\n\r\t\u1680\u180e\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000\ufeff\x09\x0a\x0b\x0c\x0d\x20\xa0]+$/ // eslint-disable-line - -/** - * Collapse multiple consecutive line breaks into single newlines, - * and remove lines that are entirely whitespace. - * - * @param {string} str - Input string - * @returns {string} Cleaned string - */ -const stripMultiLinebreaks = (str) => { - return str.replace(/(\r\n|\n|\u2424){2,}/g, '\n').split('\n').map((line) => { - return WS_REGEXP.test(line) ? line.trim() : line - }).filter((line) => { - return line.length > 0 - }).join('\n') -} - -/** - * Replace all-whitespace sequences with a single space. - * - * @param {string} str - Input string - * @returns {string} Cleaned string - */ -const stripMultispaces = (str) => { - return str.replace(WS_REGEXP, ' ').trim() -} - -/** - * Detect HTML character encoding from meta tags. - * - * @param {string} html - Raw HTML content - * @returns {string} Charset name (defaults to 'utf8') - */ -export const getCharset = (html) => { - const doc = new DOMParser().parseFromString(html, 'text/html') - const m = doc.querySelector('meta[charset]') || null - let charset = m ? m.getAttribute('charset') : '' - if (!charset) { - const h = doc.querySelector('meta[http-equiv="content-type"]') || null - charset = h ? h.getAttribute('content')?.split(';')[1]?.replace('charset=', '')?.trim() : '' - } - return charset?.toLowerCase() || 'utf8' -} - -/** - * Final cleanup of extracted article content: - * sanitize to allowed tags, collapse whitespace. - * - * @param {string} inputHtml - Extracted article HTML - * @returns {string} Cleaned HTML string - */ -export const cleanify = (inputHtml) => { - const doc = new DOMParser().parseFromString(inputHtml, 'text/html') - const html = doc.documentElement.innerHTML - return pipe( - input => sanitize(input, getSanitizeHtmlOptions()), - input => stripMultiLinebreaks(input), - input => stripMultispaces(input) - )(html) -} - -/** - * Count the number of img tags in HTML content. - * - * @param {string} html - HTML content - * @returns {number} Number of img elements - */ -export const countImages = (html) => { - const doc = new DOMParser().parseFromString(html, 'text/html') - const imgTags = doc.querySelectorAll('img') || [] - return imgTags.length -} diff --git a/src/utils/html.test.js b/src/utils/html.test.js deleted file mode 100644 index f60eb71..0000000 --- a/src/utils/html.test.js +++ /dev/null @@ -1,23 +0,0 @@ -// html.test -import { describe, it } from 'node:test' -import assert from 'node:assert' - -import { readFileSync } from 'node:fs' - -import { isString } from '@pwshub/bellajs' - -import { - cleanify -} from './html.js' - -describe('test cleanify() method', () => { - it('check if unwanted elements/attributes removed', () => { - const html = readFileSync('./test-data/regular-article.html', 'utf8') - assert.ok(html.includes('
4746 Kelly Drive, West Virginia
')) - assert.ok(html.includes('')) - const result = cleanify(html) - assert.ok(isString(result)) - assert.equal(result.includes('
4746 Kelly Drive, West Virginia
'), false) - assert.equal(result.includes(''), false) - }) -}) diff --git a/src/utils/html.ts b/src/utils/html.ts new file mode 100644 index 0000000..0b5b542 --- /dev/null +++ b/src/utils/html.ts @@ -0,0 +1,57 @@ +// utils -> html.ts + +import { DOMParser } from "linkedom"; +import sanitize from "sanitize-html"; +import { pipe } from "@pwshub/bellajs"; + +import { getSanitizeHtmlOptions } from "../config.ts"; + +export const purify = (html: string): string => { + return sanitize(html, { + allowedTags: false, + allowedAttributes: false, + allowVulnerableTags: true, + }); +}; + +// deno-lint-ignore no-control-regex +const WS_REGEXP = /^[\s\f\n\r\t\u1680\u180e\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000\ufeff\x09\x0a\x0b\x0c\x0d\x20\xa0]+$/; + +const stripMultiLinebreaks = (str: string): string => { + return str.replace(/(\r\n|\n|\u2424){2,}/g, "\n").split("\n").map((line) => { + return WS_REGEXP.test(line) ? line.trim() : line; + }).filter((line) => { + return line.length > 0; + }).join("\n"); +}; + +const stripMultispaces = (str: string): string => { + return str.replace(WS_REGEXP, " ").trim(); +}; + +export const getCharset = (html: string): string => { + const doc = new DOMParser().parseFromString(html, "text/html"); + const m = doc.querySelector("meta[charset]") || null; + let charset = m ? m.getAttribute("charset") : ""; + if (!charset) { + const h = doc.querySelector('meta[http-equiv="content-type"]') || null; + charset = h ? h.getAttribute("content")?.split(";")[1]?.replace("charset=", "")?.trim() : ""; + } + return charset?.toLowerCase() || "utf8"; +}; + +export const cleanify = (inputHtml: string): string => { + const doc = new DOMParser().parseFromString(inputHtml, "text/html"); + const html = doc.documentElement.innerHTML; + return pipe( + (input: string) => sanitize(input, getSanitizeHtmlOptions() as sanitize.IOptions), + (input: string) => stripMultiLinebreaks(input), + (input: string) => stripMultispaces(input), + )(html); +}; + +export const countImages = (html: string): number => { + const doc = new DOMParser().parseFromString(html, "text/html"); + const imgTags = doc.querySelectorAll("img") || []; + return imgTags.length; +}; diff --git a/src/utils/linker.js b/src/utils/linker.js deleted file mode 100644 index 59ac04f..0000000 --- a/src/utils/linker.js +++ /dev/null @@ -1,185 +0,0 @@ -// utils -> linker - -import { DOMParser } from 'linkedom' - -import { findBestMatch } from './similarity.js' - -/** - * Check if a string is a valid HTTP or HTTPS URL. - * - * @param {string} [url=''] - URL string to validate - * @returns {boolean} True if valid HTTP(S) URL - */ -export const isValid = (url = '') => { - try { - const ourl = new URL(url) - return ourl !== null && ourl.protocol.startsWith('http') - } catch { - return false - } -} - -/** - * Pick the URL that best matches the article title using string similarity. - * - * @param {string[]} [candidates=[]] - Candidate URLs - * @param {string} [title=''] - Article title for comparison - * @returns {string} Best matching URL - */ -export const chooseBestUrl = (candidates = [], title = '') => { - const ranking = findBestMatch(title, candidates) - return ranking.bestMatch.target -} - -/** - * Resolve a relative URL against a base URL. - * - * @param {string} [fullUrl=''] - Base URL - * @param {string} [relativeUrl=''] - Relative URL to resolve - * @returns {string} Absolute URL or empty string on failure - */ -export const absolutify = (fullUrl = '', relativeUrl = '') => { - try { - const result = new URL(relativeUrl, fullUrl) - return result.toString() - } catch { - return '' - } -} - -/** - * Tracking and analytics query parameters to strip from URLs. - * - * @type {string[]} - */ -const blacklistKeys = [ - 'CNDID', - '__twitter_impression', - '_hsenc', - '_openstat', - 'action_object_map', - 'action_ref_map', - 'action_type_map', - 'amp', - 'fb_action_ids', - 'fb_action_types', - 'fb_ref', - 'fb_source', - 'fbclid', - 'ga_campaign', - 'ga_content', - 'ga_medium', - 'ga_place', - 'ga_source', - 'ga_term', - 'gs_l', - 'hmb_campaign', - 'hmb_medium', - 'hmb_source', - 'mbid', - 'mc_cid', - 'mc_eid', - 'mkt_tok', - 'referrer', - 'spJobID', - 'spMailingID', - 'spReportId', - 'spUserID', - 'utm_brand', - 'utm_campaign', - 'utm_cid', - 'utm_content', - 'utm_int', - 'utm_mailing', - 'utm_medium', - 'utm_name', - 'utm_place', - 'utm_pubreferrer', - 'utm_reader', - 'utm_social', - 'utm_source', - 'utm_swu', - 'utm_term', - 'utm_userid', - 'utm_viz_id', - 'wt_mc_o', - 'yclid', - 'WT.mc_id', - 'WT.mc_ev', - 'WT.srch', - 'pk_source', - 'pk_medium', - 'pk_campaign', -] - -/** - * Remove tracking parameters and hash fragment from a URL. - * - * @param {string} url - URL to clean - * @returns {string|null} Cleaned URL or null if invalid - */ -export const purify = (url) => { - try { - const pureUrl = new URL(url) - - blacklistKeys.forEach((key) => { - pureUrl.searchParams.delete(key) - }) - - return pureUrl.toString().replace(pureUrl.hash, '') - } catch { - return null - } -} - -/** - * @param inputHtml {string} - * @param url {string} - * @returns article {string} - */ -/** - * Normalize all links, images, and source elements in HTML - * by resolving relative URLs to absolute and adding target=_blank to links. - * - * @param {string} html - HTML content to normalize - * @param {string} url - Base URL for resolving relative paths - * @returns {string} Normalized HTML string - */ -export const normalize = (html, url) => { - const doc = new DOMParser().parseFromString(html, 'text/html') - - Array.from(doc.getElementsByTagName('a')).forEach((element) => { - const href = element.getAttribute('href') - if (href) { - element.setAttribute('href', absolutify(url, href)) - element.setAttribute('target', '_blank') - } - }) - - Array.from(doc.getElementsByTagName('img')).forEach((element) => { - const src = element.getAttribute('data-src') ?? element.getAttribute('src') - if (src) { - element.setAttribute('src', absolutify(url, src)) - } - }) - - Array.from(doc.getElementsByTagName('source')).forEach((element) => { - const src = element.getAttribute('src') - if (src) { - element.setAttribute('src', absolutify(url, src)) - } - }) - - return Array.from(doc.childNodes).map(element => element.outerHTML).join('') -} - -/** - * Extract the domain from a URL, stripping the www. prefix. - * - * @param {string} url - Full URL - * @returns {string} Domain name - */ -export const getDomain = (url) => { - const host = (new URL(url)).host - return host.replace('www.', '') -} diff --git a/src/utils/linker.test.js b/src/utils/linker.test.js deleted file mode 100644 index 89eac3c..0000000 --- a/src/utils/linker.test.js +++ /dev/null @@ -1,182 +0,0 @@ -// linker.test -import { describe, it } from 'node:test' -import assert from 'node:assert' - -import { readFileSync } from 'node:fs' - -import { isString } from '@pwshub/bellajs' - -import { - chooseBestUrl, - isValid as isValidUrl, - purify as purifyUrl, - normalize as normalizeUrls, - absolutify as absolutifyUrl -} from './linker.js' - -describe('test isValidUrl()', () => { - const cases = [ - { - url: 'https://www.23hq.com', - expected: true, - }, - { - url: 'https://secure.actblue.com', - expected: true, - }, - { - url: 'https://docs.microsoft.com/en-us/azure/iot-edge/quickstart?view=iotedge-2018-06', - expected: true, - }, - { - url: 'http://192.168.1.199:8081/example/page', - expected: true, - }, - { - url: 'ftp://192.168.1.199:8081/example/page', - expected: false, - }, - { - url: '', - expected: false, - }, - { - url: null, - expected: false, - }, - { - url: { a: 'x' }, - expected: false, - }, - ] - cases.forEach(({ url, expected }) => { - it(`isValidUrl("${url}") must return "${expected}"`, () => { - const result = isValidUrl(url) - assert.equal(result, expected) - }) - }) -}) - -describe('test normalizeUrls()', () => { - it('test adding absolute URLs to all links', () => { - const bestUrl = 'https://test-url.com/burritos-for-life' - const html = readFileSync('./test-data/regular-article.html', 'utf8') - const result = normalizeUrls(html, bestUrl) - assert.ok(isString(result)) - assert.equal(result.includes('watermelon'), false) - assert.equal(result.includes('watermelon'), true) - }) - it('test adding target=_blank to all links', () => { - const bestUrl = 'https://test-url.com/burritos-for-life' - const html = readFileSync('./test-data/regular-article.html', 'utf8') - const result = normalizeUrls(html, bestUrl) - assert.ok(isString(result)) - assert.equal(result.includes('rational peach'), false) - assert.equal(result.includes('rational peach'), true) - }) -}) - -describe('test purifyUrl()', () => { - const entries = [ - { - url: '', - expected: null, - }, - { - url: {}, - expected: null, - }, - { - url: 'https://some.where/article/abc-xyz', - expected: 'https://some.where/article/abc-xyz', - }, - { - url: 'https://some.where/article/abc-xyz#name,bob', - expected: 'https://some.where/article/abc-xyz', - }, - { - url: 'https://some.where/article/abc-xyz?utm_source=news4&utm_medium=email&utm_campaign=spring-summer', - expected: 'https://some.where/article/abc-xyz', - }, - { - url: 'https://some.where/article/abc-xyz?q=3&utm_source=news4&utm_medium=email&utm_campaign=spring-summer', - expected: 'https://some.where/article/abc-xyz?q=3', - }, - { - url: 'https://some.where/article/abc-xyz?pk_source=news4&pk_medium=email&pk_campaign=spring-summer', - expected: 'https://some.where/article/abc-xyz', - }, - { - url: 'https://some.where/article/abc-xyz?q=3&pk_source=news4&pk_medium=email&pk_campaign=spring-summer', - expected: 'https://some.where/article/abc-xyz?q=3', - }, - ] - entries.forEach((entry) => { - const { - url, - expected, - } = entry - it(`purifyUrl("${url}") must become "${expected}"`, () => { - const result = purifyUrl(url) - assert.equal(result, expected) - }) - }) -}) - -describe('test absolutifyUrl()', () => { - const entries = [ - { - full: '', - expected: '', - }, - { - relative: {}, - expected: '', - }, - { - full: 'https://some.where/article/abc-xyz', - relative: 'category/page.html', - expected: 'https://some.where/article/category/page.html', - }, - { - full: 'https://some.where/article/abc-xyz', - relative: '../category/page.html', - expected: 'https://some.where/category/page.html', - }, - { - full: 'https://some.where/blog/authors/article/abc-xyz', - relative: '/category/page.html', - expected: 'https://some.where/category/page.html', - }, - { - full: 'https://some.where/article/abc-xyz', - expected: 'https://some.where/article/abc-xyz', - }, - ] - entries.forEach((entry) => { - const { - full, - relative, - expected, - } = entry - it(`absolutifyUrl("${full}", "${relative}") must become "${expected}"`, () => { - const result = absolutifyUrl(full, relative) - assert.equal(result, expected) - }) - }) -}) - -describe('test chooseBestUrl()', () => { - it('test chooseBestUrl an actual case', () => { - const title = 'Google Ä‘ÃŖ ra giÃĄ mua Fitbit' - const urls = [ - 'https://alpha.xyz/tin-tuc-kinh-doanh/-/view_content/content/2965950/google-da-ra-gia-mua-fitbit', - 'https://alpha.xyz/tin-tuc-kinh-doanh/view/2965950/907893219797', - 'https://alpha.xyz/tin-tuc-kinh-doanh/google-da-ra-gia-mua-fitbit', - 'https://a.xyz/read/google-da-ra-gia-mua-fitbit', - 'https://a.xyz/read/2965950/907893219797', - ] - const result = chooseBestUrl(urls, title) - assert.equal(result, urls[3]) - }) -}) diff --git a/src/utils/linker.ts b/src/utils/linker.ts new file mode 100644 index 0000000..79fc2c8 --- /dev/null +++ b/src/utils/linker.ts @@ -0,0 +1,135 @@ +// utils -> linker.ts + +import { DOMParser } from "linkedom"; + +import { findBestMatch } from "./similarity.ts"; + +export const isValid = (url: string = ""): boolean => { + try { + const ourl = new URL(url); + return ourl !== null && ourl.protocol.startsWith("http"); + } catch { + return false; + } +}; + +export const chooseBestUrl = (candidates: string[] = [], title = ""): string => { + const ranking = findBestMatch(title, candidates); + return ranking.bestMatch.target; +}; + +export const absolutify = (fullUrl = "", relativeUrl = ""): string => { + try { + const result = new URL(relativeUrl, fullUrl); + return result.toString(); + } catch { + return ""; + } +}; + +const blacklistKeys = [ + "CNDID", + "__twitter_impression", + "_hsenc", + "_openstat", + "action_object_map", + "action_ref_map", + "action_type_map", + "amp", + "fb_action_ids", + "fb_action_types", + "fb_ref", + "fb_source", + "fbclid", + "ga_campaign", + "ga_content", + "ga_medium", + "ga_place", + "ga_source", + "ga_term", + "gs_l", + "hmb_campaign", + "hmb_medium", + "hmb_source", + "mbid", + "mc_cid", + "mc_eid", + "mkt_tok", + "referrer", + "spJobID", + "spMailingID", + "spReportId", + "spUserID", + "utm_brand", + "utm_campaign", + "utm_cid", + "utm_content", + "utm_int", + "utm_mailing", + "utm_medium", + "utm_name", + "utm_place", + "utm_pubreferrer", + "utm_reader", + "utm_social", + "utm_source", + "utm_swu", + "utm_term", + "utm_userid", + "utm_viz_id", + "wt_mc_o", + "yclid", + "WT.mc_id", + "WT.mc_ev", + "WT.srch", + "pk_source", + "pk_medium", + "pk_campaign", +]; + +export const purify = (url: string): string | null => { + try { + const pureUrl = new URL(url); + + blacklistKeys.forEach((key) => { + pureUrl.searchParams.delete(key); + }); + + return pureUrl.toString().replace(pureUrl.hash, ""); + } catch { + return null; + } +}; + +export const normalize = (html: string, url: string): string => { + const doc = new DOMParser().parseFromString(html, "text/html"); + + Array.from(doc.getElementsByTagName("a")).forEach((element) => { + const href = element.getAttribute("href"); + if (href) { + element.setAttribute("href", absolutify(url, href)); + element.setAttribute("target", "_blank"); + } + }); + + Array.from(doc.getElementsByTagName("img")).forEach((element) => { + const src = element.getAttribute("data-src") ?? element.getAttribute("src"); + if (src) { + element.setAttribute("src", absolutify(url, src)); + } + }); + + Array.from(doc.getElementsByTagName("source")).forEach((element) => { + const src = element.getAttribute("src"); + if (src) { + element.setAttribute("src", absolutify(url, src)); + } + }); + + return Array.from(doc.childNodes).map((element) => (element as Element).outerHTML).join(""); +}; + +export const getDomain = (url: string): string => { + const host = (new URL(url)).host; + return host.replace("www.", ""); +}; diff --git a/src/utils/parseFromHtml.js b/src/utils/parseFromHtml.js deleted file mode 100644 index 2df173b..0000000 --- a/src/utils/parseFromHtml.js +++ /dev/null @@ -1,163 +0,0 @@ -// utils -> parseFromHtml - -import { - stripTags, - truncateByChar, - unique, - pipe, - getTTR -} from '@pwshub/bellajs' - -import { - purify, - cleanify, - countImages -} from './html.js' - -import { - isValid as isValidUrl, - purify as purifyUrl, - absolutify as absolutifyUrl, - normalize as normalizeUrls, - chooseBestUrl, - getDomain -} from './linker.js' - -import extractMetaData from './extractMetaData.js' - -import extractWithReadability, { - extractTitleWithReadability -} from './extractWithReadability.js' - -import { execPreParser, execPostParser } from './transformation.js' - -/** - * Build article description from meta description or text content. - * - * @param {Object} params - * @param {string} params.desc - Meta description - * @param {string} params.text - Stripped text content - * @param {number} params.threshold - Min length to use meta description - * @param {number} params.maxlen - Max chars for truncated description - * @returns {string} Final description string - */ -const summarize = ({ desc, text, threshold, maxlen }) => { - return desc.length > threshold - ? desc - : truncateByChar(text, maxlen).replace(/\n/g, ' ') -} - -/** - * Parse HTML content and extract article data. - * Orchestrates metadata extraction, URL normalization, transformations, - * Readability extraction, and content sanitization. - * - * @param {string} inputHtml - Raw HTML content - * @param {string} [inputUrl=''] - Source URL for resolving relative links - * @param {ParserOptions} [parserOptions={}] - Parsing options - * @returns {Promise} Extracted article data or null - */ -export default async (inputHtml, inputUrl = '', parserOptions = {}) => { - const pureHtml = purify(inputHtml) - const meta = extractMetaData(pureHtml) - - let title = meta.title - - const { - url, - shortlink, - amphtml, - canonical, - description: metaDesc, - image: metaImg, - author, - published, - favicon: metaFav, - type, - } = meta - - const { - wordsPerMinute = 300, - descriptionTruncateLen = 210, - descriptionLengthThreshold = 180, - contentLengthThreshold = 200, - } = parserOptions - - // gather title - if (!title) { - title = extractTitleWithReadability(pureHtml, inputUrl) - } - if (!title) { - return null - } - - // gather urls to choose the best url later - const links = unique( - [url, shortlink, amphtml, canonical, inputUrl] - .filter(isValidUrl) - .map(purifyUrl) - ) - - if (!links.length) { - return null - } - - // choose the best url, which one looks like title the most - const bestUrl = chooseBestUrl(links, title) - - const fns = pipe( - (input) => { - return normalizeUrls(input, bestUrl) - }, - (input) => { - return execPreParser(input, links) - }, - (input) => { - return extractWithReadability(input, bestUrl) - }, - (input) => { - return input ? execPostParser(input, links) : null - }, - (input) => { - return input ? cleanify(input) : null - } - ) - - const content = fns(inputHtml) - - if (!content) { - return null - } - - const textContent = stripTags(content) - if (textContent.length < contentLengthThreshold) { - return null - } - - const description = summarize({ - desc: metaDesc, - text: textContent, - threshold: descriptionLengthThreshold, - maxlen: descriptionTruncateLen, - }) - - const image = metaImg ? absolutifyUrl(bestUrl, metaImg) : '' - const favicon = metaFav ? absolutifyUrl(bestUrl, metaFav) : '' - - const imgcount = countImages(content) - - return { - url: bestUrl, - title, - description, - links, - image, - content, - author, - favicon, - source: getDomain(bestUrl), - published, - ttr: getTTR(textContent, imgcount, wordsPerMinute), - type, - } -} diff --git a/src/utils/parseFromHtml.test.js b/src/utils/parseFromHtml.test.js deleted file mode 100644 index 54512ce..0000000 --- a/src/utils/parseFromHtml.test.js +++ /dev/null @@ -1,133 +0,0 @@ -// parseFromHtml.test -import { describe, it } from 'node:test' -import assert from 'node:assert' - -import { readFileSync } from 'node:fs' - -import { isFunction } from '@pwshub/bellajs' - -import { extractFromHtml as parseFromHtml } from '../main.js' -import { addTransformations } from './transformation.js' - -const expDesc = [ - 'Navigation here Few can name a rational peach that isn\'t a conscientious goldfish!', - 'One cannot separate snakes from plucky pomegranates?', - 'Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs.', -].join(' ') - -const cases = [ - { - input: { - desc: 'a webpage with no title', - html: readFileSync('./test-data/html-no-title.html', 'utf8'), - }, - expectation: null, - }, - { - input: { - desc: 'a webpage without link', - html: readFileSync('./test-data/html-no-link.html', 'utf8'), - }, - expectation: null, - }, - { - input: { - desc: 'a webpage with no main article', - html: readFileSync('./test-data/html-no-article.html', 'utf8'), - }, - expectation: null, - }, - { - input: { - desc: 'a webpage with a very short article', - html: readFileSync('./test-data/html-too-short-article.html', 'utf8'), - url: 'abcd', - }, - expectation: null, - }, - { - input: { - desc: 'a webpage with article but no source', - html: readFileSync('./test-data/html-article-no-source.html', 'utf8'), - }, - expectation: (result) => { - assert.equal(result.source, 'somewhere.any') - }, - }, - { - input: { - desc: 'a webpage with data-src in img tag', - html: readFileSync('./test-data/html-article-with-data-src.html', 'utf8'), - }, - expectation: (result) => { - assert.equal(result.content.includes(''), true) - assert.equal(result.content.includes(''), true) - }, - }, - { - input: { - desc: 'a webpage with regular article', - html: readFileSync('./test-data/regular-article.html', 'utf8'), - url: 'https://somewhere.com/path/to/article', - }, - expectation: (result) => { - assert.equal(result.title, 'Article title here') - assert.equal(result.description, expDesc) - assert.equal(result.content.includes(''), true) - assert.equal(result.content.includes(''), true) - }, - }, -] - -describe('test parseFromHtml', () => { - cases.forEach((acase) => { - const { input, expectation } = acase - const { desc, html, url = '' } = input - it(`check if parseFromHtml() works with ${desc}`, async () => { - const result = await parseFromHtml(html, url) - if (isFunction(expectation)) { - expectation(result) - } else { - assert.equal(result, expectation) - } - }) - }) - - it('check if parseFromHtml() works with multi transforms', async () => { - addTransformations([ - { - patterns: [ - /http(s?):\/\/need-transform.tld\/*/, - ], - post: (document) => { - document.querySelectorAll('a').forEach((node) => { - const sHtml = node.innerHTML - const link = node.getAttribute('href') - node.parentNode.replaceChild(document.createTextNode(`[link url="${link}"]${sHtml}[/link]`), node) - }) - return document - }, - }, - { - patterns: [ - /http(s?):\/\/sw.re\/*/, - ], - post: (document) => { - document.querySelectorAll('strong').forEach((node) => { - const b = document.createElement('B') - b.innerHTML = node.innerHTML - node.parentNode.replaceChild(b, node) - }) - return document - }, - }, - ]) - const html = readFileSync('./test-data/vnn-article.html', 'utf8') - const url = 'https://need-transform.tld/path/to/article' - const result = await parseFromHtml(html, url) - assert.equal(result.title, 'Article title here') - assert.equal(result.content.includes(''), false) - assert.equal(result.content.includes('[link url="https://vnn.vn/dict/watermelon"]watermelon[/link]'), true) - assert.equal(result.content.includes('in its own way'), true) - }) -}) diff --git a/src/utils/parseFromHtml.ts b/src/utils/parseFromHtml.ts new file mode 100644 index 0000000..f90bde2 --- /dev/null +++ b/src/utils/parseFromHtml.ts @@ -0,0 +1,143 @@ +// utils -> parseFromHtml.ts + +import { + stripTags, + truncateByChar, + unique, + pipe, + getTTR, +} from "@pwshub/bellajs"; + +import { + purify, + cleanify, + countImages, +} from "./html.ts"; + +import { + isValid as isValidUrl, + purify as purifyUrl, + absolutify as absolutifyUrl, + normalize as normalizeUrls, + chooseBestUrl, + getDomain, +} from "./linker.ts"; + +import extractMetaData from "./extractMetaData.ts"; + +import extractWithReadability, { + extractTitleWithReadability, +} from "./extractWithReadability.ts"; + +import { execPreParser, execPostParser } from "./transformation.ts"; + +import type { ArticleData, ParserOptions } from "../main.ts"; + +const summarize = ({ desc, text, threshold, maxlen }: { desc: string; text: string; threshold: number; maxlen: number }): string => { + return desc.length > threshold + ? desc + : truncateByChar(text, maxlen).replace(/\n/g, " "); +}; + +// deno-lint-ignore require-await +export default async (inputHtml: string, inputUrl = "", parserOptions: ParserOptions = {}): Promise => { + const pureHtml = purify(inputHtml); + const meta = extractMetaData(pureHtml); + + let title = meta.title; + + const { + url, + shortlink, + amphtml, + canonical, + description: metaDesc, + image: metaImg, + author, + published, + favicon: metaFav, + type, + } = meta; + + const { + wordsPerMinute = 300, + descriptionTruncateLen = 210, + descriptionLengthThreshold = 180, + contentLengthThreshold = 200, + } = parserOptions; + + if (!title) { + title = extractTitleWithReadability(pureHtml) || ""; + } + if (!title) { + return null; + } + + const links = unique( + [url, shortlink, amphtml, canonical, inputUrl] + .filter(isValidUrl) + .map(purifyUrl as (s: string) => string), + ); + + if (!links.length) { + return null; + } + + const bestUrl = chooseBestUrl(links, title); + + const fns = pipe( + (input: string) => { + return normalizeUrls(input, bestUrl); + }, + (input: string) => { + return execPreParser(input, links); + }, + (input: string) => { + return extractWithReadability(input, bestUrl); + }, + (input: string | null) => { + return input ? execPostParser(input, links) : null; + }, + (input: string | null) => { + return input ? cleanify(input) : null; + }, + ); + + const content = fns(inputHtml); + + if (!content) { + return null; + } + + const textContent = stripTags(content); + if (textContent.length < contentLengthThreshold) { + return null; + } + + const description = summarize({ + desc: metaDesc, + text: textContent, + threshold: descriptionLengthThreshold, + maxlen: descriptionTruncateLen, + }); + + const image = metaImg ? absolutifyUrl(bestUrl, metaImg) : ""; + const favicon = metaFav ? absolutifyUrl(bestUrl, metaFav) : ""; + + const imgcount = countImages(content); + + return { + url: bestUrl, + title, + description, + links, + image, + content, + author, + favicon, + source: getDomain(bestUrl), + published, + ttr: getTTR(textContent, imgcount, wordsPerMinute), + type, + }; +}; diff --git a/src/utils/retrieve.js b/src/utils/retrieve.js deleted file mode 100644 index a00d8c4..0000000 --- a/src/utils/retrieve.js +++ /dev/null @@ -1,19 +0,0 @@ -// utils -> retrieve - -/** - * Fetch content from a URL using the provided fetcher function. - * - * @param {string} url - URL to fetch - * @param {Function} fetcher - Custom fetch function (url) => Promise - * @returns {Promise} Response body as ArrayBuffer - */ -export default async (url, fetcher) => { - const res = await fetcher(url) - - const status = res.status - if (status >= 400) { - throw new Error(`Request failed with error code ${status}`) - } - const buffer = await res.arrayBuffer() - return buffer -} diff --git a/src/utils/retrieve.test.js b/src/utils/retrieve.test.js deleted file mode 100644 index 0e95e3b..0000000 --- a/src/utils/retrieve.test.js +++ /dev/null @@ -1,65 +0,0 @@ -// retrieve.test -import { describe, it } from 'node:test' -import assert from 'node:assert' - -import nock from 'nock' - -import retrieve from './retrieve.js' - -const parseUrl = (url) => { - const re = new URL(url) - return { - baseUrl: `${re.protocol}//${re.host}`, - path: re.pathname, - } -} - -describe('test retrieve() method', () => { - it('test retrieve with bad status code', async () => { - const url = 'https://some.where/bad/page' - const { baseUrl, path } = parseUrl(url) - nock(baseUrl).get(path).reply(500, 'Error 500') - assert.rejects(retrieve(url, globalThis.fetch), new Error('Request failed with error code 500')) - }) - - it('test retrieve from good source', async () => { - const url = 'https://some.where/good/page' - const { baseUrl, path } = parseUrl(url) - nock(baseUrl).get(path).reply(200, '
this is content
', { - 'Content-Type': 'text/html', - }) - const buffer = await retrieve(url, globalThis.fetch) - const html = Buffer.from(buffer).toString() - assert.equal(html, '
this is content
') - }) - - it('test retrieve from good source with \\r\\n', async () => { - const url = 'https://some.where/good/page' - const { baseUrl, path } = parseUrl(url) - nock(baseUrl).get(path).reply(200, '\n\r\r\n\n
this is content
\n\r\r\n\n', { - 'Content-Type': 'text/html', - }) - const buffer = await retrieve(url, globalThis.fetch) - const html = Buffer.from(buffer).toString().trim() - assert.equal(html, '
this is content
') - }) - - it('test retrieve with custom fetcher', async () => { - const url = 'https://some.where/good/source-with-fetcher' - nock('https://proxy-server.com') - .get('/api/proxy?url=https%3A%2F%2Fsome.where%2Fgood%2Fsource-with-fetcher') - .reply(200, '
this is content
', { - 'Content-Type': 'text/html', - }) - - const myFetcher = (fetchUrl) => { - const proxyUrl = 'https://proxy-server.com/api/proxy?url=' + encodeURIComponent(fetchUrl) - return globalThis.fetch(proxyUrl) - } - - const buffer = await retrieve(url, myFetcher) - const html = Buffer.from(buffer).toString() - assert.equal(html, '
this is content
') - nock.cleanAll() - }) -}) diff --git a/src/utils/retrieve.ts b/src/utils/retrieve.ts new file mode 100644 index 0000000..ca894f4 --- /dev/null +++ b/src/utils/retrieve.ts @@ -0,0 +1,12 @@ +// utils -> retrieve.ts + +export default async (url: string, fetcher: (url: string) => Promise): Promise => { + const res = await fetcher(url); + + const status = res.status; + if (status >= 400) { + throw new Error(`Request failed with error code ${status}`); + } + const buffer = await res.arrayBuffer(); + return buffer; +}; diff --git a/src/utils/similarity.js b/src/utils/similarity.js deleted file mode 100644 index 4201ca9..0000000 --- a/src/utils/similarity.js +++ /dev/null @@ -1,48 +0,0 @@ -// similarity.js - -import { - isString, - compareTwoStrings, - isArray -} from '@pwshub/bellajs' - -/** - * Validate arguments for findBestMatch. - * - * @param {string} mainString - Reference string - * @param {string[]} targetStrings - Strings to compare against - * @returns {boolean} True if arguments are valid - */ -const areArgsValid = (mainString, targetStrings) => { - return isString(mainString) && isArray(targetStrings) - && targetStrings.length > 0 && targetStrings.every(s => isString(s)) -} - -/** - * Find the best matching string from a list using Dice coefficient. - * - * @param {string} mainString - Reference string to match against - * @param {string[]} targetStrings - Candidate strings - * @returns {{ratings: Array, bestMatch: {target: string, rating: number}, bestMatchIndex: number}} Match results with rankings - */ -export const findBestMatch = (mainString, targetStrings) => { - if (!areArgsValid(mainString, targetStrings)) { - throw new Error('Bad arguments: First argument should be a string, second should be an array of strings') - } - - const ratings = [] - let bestMatchIndex = 0 - - for (let i = 0; i < targetStrings.length; i++) { - const currentTargetString = targetStrings[i] - const currentRating = compareTwoStrings(mainString, currentTargetString) - ratings.push({ target: currentTargetString, rating: currentRating }) - if (currentRating > ratings[bestMatchIndex].rating) { - bestMatchIndex = i - } - } - - const bestMatch = ratings[bestMatchIndex] - - return { ratings: ratings, bestMatch: bestMatch, bestMatchIndex: bestMatchIndex } -} diff --git a/src/utils/similarity.ts b/src/utils/similarity.ts new file mode 100644 index 0000000..be6a102 --- /dev/null +++ b/src/utils/similarity.ts @@ -0,0 +1,45 @@ +// utils -> similarity.ts + +import { + isString, + compareTwoStrings, + isArray, +} from "@pwshub/bellajs"; + +interface MatchResult { + target: string; + rating: number; +} + +interface BestMatchResult { + ratings: MatchResult[]; + bestMatch: MatchResult; + bestMatchIndex: number; +} + +const areArgsValid = (mainString: unknown, targetStrings: unknown): boolean => { + return isString(mainString) && isArray(targetStrings) && + (targetStrings as unknown[]).length > 0 && (targetStrings as string[]).every((s) => isString(s)); +}; + +export const findBestMatch = (mainString: string, targetStrings: string[]): BestMatchResult => { + if (!areArgsValid(mainString, targetStrings)) { + throw new Error("Bad arguments: First argument should be a string, second should be an array of strings"); + } + + const ratings: MatchResult[] = []; + let bestMatchIndex = 0; + + for (let i = 0; i < targetStrings.length; i++) { + const currentTargetString = targetStrings[i]; + const currentRating = compareTwoStrings(mainString, currentTargetString); + ratings.push({ target: currentTargetString, rating: currentRating }); + if (currentRating > ratings[bestMatchIndex].rating) { + bestMatchIndex = i; + } + } + + const bestMatch = ratings[bestMatchIndex]; + + return { ratings, bestMatch, bestMatchIndex }; +}; diff --git a/src/utils/transformation.js b/src/utils/transformation.js deleted file mode 100644 index 06aff82..0000000 --- a/src/utils/transformation.js +++ /dev/null @@ -1,122 +0,0 @@ -// utils --> transformation.js - -import { isArray, isFunction } from '@pwshub/bellajs' -import { DOMParser } from 'linkedom' - -/** - * Registered transformation rules for per-site HTML pre/post processing. - * - * @type {Transformation[]} - */ -const transformations = [] - -/** - * Add a single transformation to the registry. - * - * @param {Transformation} tn - Transformation object with patterns and handlers - * @returns {number} 1 if added, 0 if invalid - */ -const add = (tn) => { - const { patterns } = tn - if (!patterns || !isArray(patterns) || !patterns.length) { - return 0 - } - transformations.push(tn) - return 1 -} - -/** - * Register one or more transformations for per-site HTML processing. - * - * @param {Transformation|Transformation[]} tfms - Transformation(s) to add - * @returns {number} Number of transformations successfully added - */ -export const addTransformations = (tfms) => { - if (isArray(tfms)) { - return tfms.map(tfm => add(tfm)).filter(result => result === 1).length - } - return add(tfms) -} - -/** - * Remove transformations matching the given patterns. - * Calling without arguments removes all transformations. - * - * @param {RegExp[]} [patterns] - URL patterns to match for removal - * @returns {number} Number of transformations removed - */ -export const removeTransformations = (patterns) => { - if (!patterns) { - const removed = transformations.length - transformations.length = 0 - return removed - } - let removing = 0 - for (let i = transformations.length - 1; i >= 0; i--) { - const { patterns: ipatterns } = transformations[i] - const matched = ipatterns.some((ptn) => patterns.some((pattern) => String(pattern) === String(ptn))) - if (matched) { - transformations.splice(i, 1) - removing += 1 - } - } - return removing -} - -/** - * Get a copy of all registered transformations. - * - * @returns {Transformation[]} Copy of transformations array - */ -export const getTransformations = () => { - return [...transformations] -} - -/** - * Find all transformations whose patterns match any of the given URLs. - * - * @param {string|string[]} links - URL(s) to match against transformation patterns - * @returns {Transformation[]} Matching transformations - */ -export const findTransformations = (links) => { - const urls = !isArray(links) ? [links] : links - const tfms = [] - for (const transformation of transformations) { - const { patterns } = transformation - const matched = urls.some((url) => patterns.some((pattern) => pattern.test(url))) - if (matched) { - tfms.push({ - ...transformation, - }) - } - } - return tfms -} - -/** - * Run pre-extraction transformations on raw HTML. - * Mutates the DOM in place through registered pre-processor functions. - * - * @param {string} html - Raw HTML content - * @param {string[]} links - URLs to match against transformation patterns - * @returns {string} Transformed HTML string - */ -export const execPreParser = (html, links) => { - const doc = new DOMParser().parseFromString(html, 'text/html') - findTransformations(links).map(tfm => tfm.pre).filter(fn => isFunction(fn)).forEach(fn => fn(doc)) - return Array.from(doc.childNodes).map(it => it.outerHTML).join('') -} - -/** - * Run post-extraction transformations on extracted article content. - * Mutates the DOM in place through registered post-processor functions. - * - * @param {string} html - Extracted article HTML - * @param {string[]} links - URLs to match against transformation patterns - * @returns {string} Transformed HTML string - */ -export const execPostParser = (html, links) => { - const doc = new DOMParser().parseFromString(html, 'text/html') - findTransformations(links).map(tfm => tfm.post).filter(fn => isFunction(fn)).forEach(fn => fn(doc)) - return Array.from(doc.childNodes).map(it => it.outerHTML).join('') -} diff --git a/src/utils/transformation.test.js b/src/utils/transformation.test.js deleted file mode 100644 index 18a54d4..0000000 --- a/src/utils/transformation.test.js +++ /dev/null @@ -1,194 +0,0 @@ -// transformation.test -import { describe, it as test } from 'node:test' -import assert from 'node:assert' - -import { - addTransformations, - removeTransformations, - getTransformations, - findTransformations, - execPreParser, - execPostParser -} from './transformation.js' - -describe('test transformation', () => { - test(' add one transformation object', () => { - const result = addTransformations({ - patterns: [ - /http(s?):\/\/([\w]+.)?def.tld\/*/, - ], - pre: (document) => { - return document - }, - post: (document) => { - return document - }, - }) - assert.equal(result, 1) - }) - - test(' add multi transformation object', () => { - const result = addTransformations([ - { - patterns: [ - /http(s?):\/\/google.com\/*/, - /http(s?):\/\/goo.gl\/*/, - ], - }, - { - patterns: [ - /http(s?):\/\/goo.gl\/*/, - /http(s?):\/\/google.inc\/*/, - ], - }, - ]) - assert.equal(result, 2) - }) - - test(' add transformation object without patterns', () => { - const result = addTransformations({ - pre: (document) => { - return document - }, - post: (document) => { - return document - }, - }) - assert.equal(result, 0) - }) - - test(' add transformation object without valid patterns', () => { - const result = addTransformations({ - patterns: 123, - pre: (document) => { - return document - }, - post: (document) => { - return document - }, - }) - assert.equal(result, 0) - }) - - test(' get all transformations', () => { - const result = getTransformations() - assert.equal(result.length, 3) - assert.deepEqual(result[0].patterns[0], /http(s?):\/\/([\w]+.)?def.tld\/*/) - }) - - test(' remove one transformation', () => { - addTransformations([ - { - patterns: [ - /http(s?):\/\/abc.com\/*/, - /http(s?):\/\/def.gl\/*/, - ], - }, - { - patterns: [ - /http(s?):\/\/hik.gl\/*/, - /http(s?):\/\/lmn.inc\/*/, - ], - }, - { - patterns: [ - /http(s?):\/\/opq.gl\/*/, - /http(s?):\/\/rst.inc\/*/, - ], - }, - ]) - const result = removeTransformations([ - /http(s?):\/\/goo.gl\/*/, - ]) - assert.equal(result, 2) - }) - - test(' get all transformations again', () => { - const result = getTransformations() - assert.equal(result.length, 4) - assert.deepEqual(result[3].patterns[1], /http(s?):\/\/rst.inc\/*/) - }) - - test(' find transformations', () => { - addTransformations([ - { - patterns: [ - /http(s?):\/\/def.gl\/*/, - /http(s?):\/\/uvw.inc\/*/, - ], - }, - ]) - const notFound = findTransformations([ - 'https://goo.gl/docs/article.html', - ]) - assert.deepEqual(notFound, []) - - const foundOne = findTransformations([ - 'https://lmn.inc/docs/article.html', - ]) - assert.equal(foundOne.length, 1) - - const foundTwo = findTransformations([ - 'https://def.gl/docs/article.html', - ]) - assert.equal(foundTwo.length, 2) - }) - - test(' run execPreParser', () => { - addTransformations([ - { - patterns: [ - /http(s?):\/\/xyz.com\/*/, - ], - pre: (doc) => { - doc.querySelectorAll('.adv').forEach((element) => { - element.parentNode.removeChild(element) - }) - return doc - }, - }, - ]) - const html = ` -
- hi user, this is an advertisement element -
free product now!
-
- ` - const result = execPreParser(html, 'https://xyz.com/article') - assert.equal(result.includes('hi user, this is an advertisement element'), true) - assert.equal(result.includes('
free product now!
'), false) - }) - - test(' run execPostParser', () => { - addTransformations([ - { - patterns: [ - /http(s?):\/\/xyz.com\/*/, - ], - post: (doc) => { - doc.querySelectorAll('b').forEach((element) => { - const itag = doc.createElement('i') - itag.innerHTML = element.innerHTML - element.parentNode.replaceChild(itag, element) - }) - return doc - }, - }, - ]) - const html = ` -
- hi user, -

Thank you for your feedback!

-
- ` - const result = execPostParser(html, 'https://xyz.com/article') - assert.equal(result.includes('user'), true) - assert.equal(result.includes('user'), false) - }) - - test(' remove all transformations', () => { - const result = removeTransformations() - assert.equal(result, 7) - assert.deepEqual(getTransformations(), []) - }) -}) diff --git a/src/utils/transformation.ts b/src/utils/transformation.ts new file mode 100644 index 0000000..2c897c6 --- /dev/null +++ b/src/utils/transformation.ts @@ -0,0 +1,83 @@ +// utils -> transformation.ts + +import { isArray, isFunction } from "@pwshub/bellajs"; +import { DOMParser } from "linkedom"; + +export interface Transformation { + patterns: RegExp[]; + pre?: (document: Document) => Document; + post?: (document: Document) => Document; +} + +const transformations: Transformation[] = []; + +const add = (tn: Transformation): number => { + const { patterns } = tn; + if (!patterns || !isArray(patterns) || !patterns.length) { + return 0; + } + transformations.push(tn); + return 1; +}; + +/** Register one or more transformations for per-site HTML processing. */ +export const addTransformations = (tfms: Transformation | Transformation[]): number => { + if (isArray(tfms)) { + return (tfms as Transformation[]).map((tfm) => add(tfm)).filter((result) => result === 1).length; + } + return add(tfms as Transformation); +}; + +/** Remove transformations matching the given patterns. Removes all if no patterns given. */ +export const removeTransformations = (patterns?: RegExp[]): number => { + if (!patterns) { + const removed = transformations.length; + transformations.length = 0; + return removed; + } + let removing = 0; + for (let i = transformations.length - 1; i >= 0; i--) { + const { patterns: ipatterns } = transformations[i]; + const matched = ipatterns.some((ptn) => + (patterns as RegExp[]).some((pattern) => String(pattern) === String(ptn)) + ); + if (matched) { + transformations.splice(i, 1); + removing += 1; + } + } + return removing; +}; + +export const getTransformations = (): Transformation[] => { + return [...transformations]; +}; + +export const findTransformations = (links: string | string[]): Transformation[] => { + const urls = !isArray(links) ? [links] : links; + const tfms: Transformation[] = []; + for (const transformation of transformations) { + const { patterns } = transformation; + const matched = (urls as string[]).some((url) => + patterns.some((pattern) => pattern.test(url)) + ); + if (matched) { + tfms.push({ + ...transformation, + }); + } + } + return tfms; +}; + +export const execPreParser = (html: string, links: string | string[]): string => { + const doc = new DOMParser().parseFromString(html, "text/html"); + findTransformations(links).map((tfm) => tfm.pre).filter((fn) => isFunction(fn)).forEach((fn) => (fn as (doc: Document) => Document)(doc as any)); + return Array.from(doc.childNodes).map((it) => (it as Element).outerHTML).join(""); +}; + +export const execPostParser = (html: string, links: string | string[]): string => { + const doc = new DOMParser().parseFromString(html, "text/html"); + findTransformations(links).map((tfm) => tfm.post).filter((fn) => isFunction(fn)).forEach((fn) => (fn as (doc: Document) => Document)(doc as any)); + return Array.from(doc.childNodes).map((it) => (it as Element).outerHTML).join(""); +}; diff --git a/tests/config.test.ts b/tests/config.test.ts new file mode 100644 index 0000000..d260d9b --- /dev/null +++ b/tests/config.test.ts @@ -0,0 +1,31 @@ +// tests/config.test.ts + +import { assertEquals } from "@std/assert"; +import { setSanitizeHtmlOptions, getSanitizeHtmlOptions } from "../src/config.ts"; + +Deno.test("setSanitizeHtmlOptions/getSanitizeHtmlOptions", () => { + setSanitizeHtmlOptions({ + allowedTags: ["div", "span"], + allowedAttributes: { + a: ["href", "title"], + }, + }); + + const actual = getSanitizeHtmlOptions(); + const actualAllowedAttributes = actual.allowedAttributes as Record; + const expectedAllowedAttributes = { + a: ["href", "title"], + }; + + assertEquals(actualAllowedAttributes, expectedAllowedAttributes); + + const actualAllowedTags = actual.allowedTags as string[]; + const expectedAllowedTags = ["div", "span"]; + assertEquals(actualAllowedTags, expectedAllowedTags); + + setSanitizeHtmlOptions({ + allowedTags: [], + }); + + assertEquals(getSanitizeHtmlOptions().allowedTags, []); +}); diff --git a/tests/extractMetaData.test.ts b/tests/extractMetaData.test.ts new file mode 100644 index 0000000..46a1b20 --- /dev/null +++ b/tests/extractMetaData.test.ts @@ -0,0 +1,72 @@ +// tests/extractMetaData.test.ts + +import { assertEquals } from "@std/assert"; +import { isObject, hasProperty } from "@pwshub/bellajs"; +import extractMetaData from "../src/utils/extractMetaData.ts"; + +const keys = "url shortlink amphtml canonical title description image author source published favicon type".split(" "); + +function isDateString(date: unknown): boolean { + if (typeof date !== "string") return false; + const d = new Date(date); + return !isNaN(d.getTime()); +} + +Deno.test("extractMetaData - good content", () => { + const html = Deno.readTextFileSync("tests/test-data/regular-article.html"); + const result = extractMetaData(html); + assertEquals(isObject(result), true); + keys.forEach((k) => { + assertEquals(hasProperty(result, k), true); + }); +}); + +Deno.test("extractMetaData - json ld schema content", () => { + const html = Deno.readTextFileSync("tests/test-data/regular-article-json-ld.html"); + const result = extractMetaData(html); + assertEquals(isObject(result), true); + keys.forEach((k) => { + assertEquals(hasProperty(result, k), true); + }); +}); + +Deno.test("extractMetaData - json ld schema fills empty meta fields", () => { + const ldJson = { + "@context": "https://schema.org", + "@type": "NewsArticle", + author: { name: "JSON Author" }, + datePublished: "2026-01-02T03:04:05Z", + description: "JSON description", + image: "https://example.com/json.jpg", + }; + const body = "

" + "Article text. ".repeat(50) + "

"; + const html = ` +Fixture article + +

Fixture article

${body}
`; + const result = extractMetaData(html); + assertEquals(result.author, "JSON Author"); + assertEquals(result.published, "2026-01-02T03:04:05Z"); + assertEquals(result.description, "JSON description"); + assertEquals(result.image, "https://example.com/json.jpg"); +}); + +Deno.test("extractMetaData - find date", () => { + const html1 = Deno.readTextFileSync("tests/test-data/regular-article-date-time.html"); + const html2 = Deno.readTextFileSync("tests/test-data/regular-article-date-itemprop.html"); + const html3 = Deno.readTextFileSync("tests/test-data/regular-article-date-span.html"); + const result1 = extractMetaData(html1); + const result2 = extractMetaData(html2); + const result3 = extractMetaData(html3); + assertEquals(isObject(result1), true); + assertEquals(isObject(result2), true); + assertEquals(isObject(result3), true); + keys.forEach((k) => { + assertEquals(hasProperty(result1, k), true); + assertEquals(hasProperty(result2, k), true); + assertEquals(hasProperty(result3, k), true); + }); + assertEquals(isDateString(result1.published), true); + assertEquals(isDateString(result2.published), true); + assertEquals(isDateString(result3.published), true); +}); diff --git a/tests/extractWithReadability.test.ts b/tests/extractWithReadability.test.ts new file mode 100644 index 0000000..c3f90eb --- /dev/null +++ b/tests/extractWithReadability.test.ts @@ -0,0 +1,36 @@ +// tests/extractWithReadability.test.ts + +import { assertEquals } from "@std/assert"; +import { isString } from "@pwshub/bellajs"; +import extractWithReadability, { extractTitleWithReadability } from "../src/utils/extractWithReadability.ts"; + +Deno.test("extractWithReadability - good html content", () => { + const html = Deno.readTextFileSync("tests/test-data/regular-article.html"); + const result = extractWithReadability(html, "https://foo.bar"); + assertEquals(isString(result), true); + assertEquals((result as string).length > 200, true); + assertEquals((result as string).includes(''), true); +}); + +Deno.test("extractWithReadability - bad html content", () => { + assertEquals(extractWithReadability(null as unknown as string), null); + assertEquals(extractWithReadability({} as unknown as string), null); + assertEquals(extractWithReadability("
"), null); +}); + +Deno.test("extractTitleWithReadability - good content", () => { + const html = Deno.readTextFileSync("tests/test-data/regular-article.html"); + const result = extractTitleWithReadability(html); + assertEquals(result, "Article title here - ArticleParser"); +}); + +Deno.test("extractTitleWithReadability - page without title", () => { + const html = Deno.readTextFileSync("tests/test-data/html-no-title.html"); + const result = extractTitleWithReadability(html); + assertEquals(result, null); +}); + +Deno.test("extractTitleWithReadability - non-string", () => { + const result = extractTitleWithReadability({} as unknown as string); + assertEquals(result, null); +}); diff --git a/tests/helpers.ts b/tests/helpers.ts new file mode 100644 index 0000000..601436b --- /dev/null +++ b/tests/helpers.ts @@ -0,0 +1,15 @@ +// tests/helpers.ts + +export const createMockFetcher = ( + body: string, + contentType: string, + status = 200, +) => { + return () => + Promise.resolve( + new Response(body, { + status, + headers: { "content-type": contentType }, + }), + ); +}; diff --git a/tests/html.test.ts b/tests/html.test.ts new file mode 100644 index 0000000..bc03e96 --- /dev/null +++ b/tests/html.test.ts @@ -0,0 +1,15 @@ +// tests/html.test.ts + +import { assertEquals } from "@std/assert"; +import { isString } from "@pwshub/bellajs"; +import { cleanify } from "../src/utils/html.ts"; + +Deno.test("cleanify - removes unwanted elements/attributes", () => { + const html = Deno.readTextFileSync("tests/test-data/regular-article.html"); + assertEquals(html.includes("
4746 Kelly Drive, West Virginia
"), true); + assertEquals(html.includes(''), true); + const result = cleanify(html); + assertEquals(isString(result), true); + assertEquals(result.includes("
4746 Kelly Drive, West Virginia
"), false); + assertEquals(result.includes(''), false); +}); diff --git a/tests/linker.test.ts b/tests/linker.test.ts new file mode 100644 index 0000000..e92a2bb --- /dev/null +++ b/tests/linker.test.ts @@ -0,0 +1,69 @@ +// tests/linker.test.ts + +import { assertEquals } from "@std/assert"; +import { isString } from "@pwshub/bellajs"; +import { + chooseBestUrl, + isValid as isValidUrl, + purify as purifyUrl, + normalize as normalizeUrls, + absolutify as absolutifyUrl, +} from "../src/utils/linker.ts"; + +Deno.test("isValidUrl - valid URLs", () => { + assertEquals(isValidUrl("https://www.23hq.com"), true); + assertEquals(isValidUrl("https://secure.actblue.com"), true); + assertEquals(isValidUrl("https://docs.microsoft.com/en-us/azure/iot-edge/quickstart?view=iotedge-2018-06"), true); + assertEquals(isValidUrl("http://192.168.1.199:8081/example/page"), true); +}); + +Deno.test("isValidUrl - invalid URLs", () => { + assertEquals(isValidUrl("ftp://192.168.1.199:8081/example/page"), false); + assertEquals(isValidUrl(""), false); + assertEquals(isValidUrl(null as unknown as string), false); + assertEquals(isValidUrl({ a: "x" } as unknown as string), false); +}); + +Deno.test("normalizeUrls - adds absolute URLs and target=_blank", () => { + const bestUrl = "https://test-url.com/burritos-for-life"; + const html = Deno.readTextFileSync("tests/test-data/regular-article.html"); + const result = normalizeUrls(html, bestUrl); + assertEquals(isString(result), true); + assertEquals(result.includes('
watermelon'), false); + assertEquals(result.includes('watermelon'), true); + assertEquals(result.includes('rational peach'), false); + assertEquals(result.includes('rational peach'), true); +}); + +Deno.test("purifyUrl - various cases", () => { + assertEquals(purifyUrl(""), null); + assertEquals(purifyUrl({} as unknown as string), null); + assertEquals(purifyUrl("https://some.where/article/abc-xyz"), "https://some.where/article/abc-xyz"); + assertEquals(purifyUrl("https://some.where/article/abc-xyz#name,bob"), "https://some.where/article/abc-xyz"); + assertEquals(purifyUrl("https://some.where/article/abc-xyz?utm_source=news4&utm_medium=email&utm_campaign=spring-summer"), "https://some.where/article/abc-xyz"); + assertEquals(purifyUrl("https://some.where/article/abc-xyz?q=3&utm_source=news4&utm_medium=email&utm_campaign=spring-summer"), "https://some.where/article/abc-xyz?q=3"); + assertEquals(purifyUrl("https://some.where/article/abc-xyz?pk_source=news4&pk_medium=email&pk_campaign=spring-summer"), "https://some.where/article/abc-xyz"); + assertEquals(purifyUrl("https://some.where/article/abc-xyz?q=3&pk_source=news4&pk_medium=email&pk_campaign=spring-summer"), "https://some.where/article/abc-xyz?q=3"); +}); + +Deno.test("absolutifyUrl - various cases", () => { + assertEquals(absolutifyUrl("", ""), ""); + assertEquals(absolutifyUrl("", {} as unknown as string), ""); + assertEquals(absolutifyUrl("https://some.where/article/abc-xyz", "category/page.html"), "https://some.where/article/category/page.html"); + assertEquals(absolutifyUrl("https://some.where/article/abc-xyz", "../category/page.html"), "https://some.where/category/page.html"); + assertEquals(absolutifyUrl("https://some.where/blog/authors/article/abc-xyz", "/category/page.html"), "https://some.where/category/page.html"); + assertEquals(absolutifyUrl("https://some.where/article/abc-xyz"), "https://some.where/article/abc-xyz"); +}); + +Deno.test("chooseBestUrl - actual case", () => { + const title = "Google Ä‘ÃŖ ra giÃĄ mua Fitbit"; + const urls = [ + "https://alpha.xyz/tin-tuc-kinh-doanh/-/view_content/content/2965950/google-da-ra-gia-mua-fitbit", + "https://alpha.xyz/tin-tuc-kinh-doanh/view/2965950/907893219797", + "https://alpha.xyz/tin-tuc-kinh-doanh/google-da-ra-gia-mua-fitbit", + "https://a.xyz/read/google-da-ra-gia-mua-fitbit", + "https://a.xyz/read/2965950/907893219797", + ]; + const result = chooseBestUrl(urls, title); + assertEquals(result, urls[3]); +}); diff --git a/tests/main.test.ts b/tests/main.test.ts new file mode 100644 index 0000000..234ff62 --- /dev/null +++ b/tests/main.test.ts @@ -0,0 +1,93 @@ +// tests/main.test.ts + +import { assertEquals, assertRejects } from "@std/assert"; +import { + extract, + getSanitizeHtmlOptions, + setSanitizeHtmlOptions, +} from "../src/main.ts"; +import { createMockFetcher } from "./helpers.ts"; + +Deno.test("extract - non-string input", async () => { + await assertRejects( + () => extract({} as unknown as string), + Error, + "Input must be a string", + ); +}); + +Deno.test("extract - bad urls", async () => { + const badSamples = [ + "htt:/abc.com/failed-none-sense", + "fpt://abc.com/failed-none-sense", + "ttp://badcom/146753785", + "https://674458092126388225", + ]; + + for (const url of badSamples) { + await assertRejects( + () => extract(url), + Error, + ); + } +}); + +Deno.test("extract - regular article from url", async () => { + const expDesc = [ + "Navigation here Few can name a rational peach that isn't a conscientious goldfish!", + "One cannot separate snakes from plucky pomegranates?", + "Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs.", + ].join(" "); + + const html = Deno.readTextFileSync("tests/test-data/regular-article.html"); + const fetcher = createMockFetcher(html, "text/html"); + const result = await extract("https://somewhere.com/path/to/article", {}, fetcher); + assertEquals((result as Record).title, "Article title here"); + assertEquals((result as Record).description, expDesc); +}); + +Deno.test("extract - no article from url", async () => { + const html = Deno.readTextFileSync("tests/test-data/html-no-article.html"); + const fetcher = createMockFetcher(html, "text/html"); + const result = await extract("https://somewhere.com/path/to/no/article", {}, fetcher); + assertEquals(result, null); +}); + +Deno.test("extract - empty content from url", async () => { + const fetcher = createMockFetcher("", "text/html"); + const result = await extract("https://somewhere.com/path/to/no/content", {}, fetcher); + assertEquals(result, null); +}); + +Deno.test("extract - html string directly", async () => { + const expDesc = [ + "Navigation here Few can name a rational peach that isn't a conscientious goldfish!", + "One cannot separate snakes from plucky pomegranates?", + "Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs.", + ].join(" "); + + const html = Deno.readTextFileSync("tests/test-data/regular-article.html"); + const result = await extract(html); + assertEquals((result as Record).title, "Article title here"); + assertEquals((result as Record).description, expDesc); +}); + +Deno.test("extract with modified sanitize-html options", async () => { + const currentSanitizeOptions = getSanitizeHtmlOptions(); + + setSanitizeHtmlOptions({ + ...currentSanitizeOptions, + allowedAttributes: { + ...currentSanitizeOptions.allowedAttributes as Record, + code: ["class"], + div: ["class"], + }, + allowedClasses: { + code: ["language-*", "lang-*"], + }, + }); + + const html = Deno.readTextFileSync("tests/test-data/article-with-classes-attributes.html"); + const result = await extract(html); + assertEquals((result as Record).content.includes('code class="lang-js"'), true); +}); diff --git a/tests/parseFromHtml.test.ts b/tests/parseFromHtml.test.ts new file mode 100644 index 0000000..da204df --- /dev/null +++ b/tests/parseFromHtml.test.ts @@ -0,0 +1,96 @@ +// tests/parseFromHtml.test.ts + +import { assertEquals } from "@std/assert"; +import { extractFromHtml as parseFromHtml } from "../src/main.ts"; +import { addTransformations, removeTransformations } from "../src/utils/transformation.ts"; + +const expDesc = [ + "Navigation here Few can name a rational peach that isn't a conscientious goldfish!", + "One cannot separate snakes from plucky pomegranates?", + "Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs.", +].join(" "); + +Deno.test("parseFromHtml - webpage with no title", async () => { + const html = Deno.readTextFileSync("tests/test-data/html-no-title.html"); + const result = await parseFromHtml(html); + assertEquals(result, null); +}); + +Deno.test("parseFromHtml - webpage without link", async () => { + const html = Deno.readTextFileSync("tests/test-data/html-no-link.html"); + const result = await parseFromHtml(html); + assertEquals(result, null); +}); + +Deno.test("parseFromHtml - webpage with no main article", async () => { + const html = Deno.readTextFileSync("tests/test-data/html-no-article.html"); + const result = await parseFromHtml(html); + assertEquals(result, null); +}); + +Deno.test("parseFromHtml - webpage with very short article", async () => { + const html = Deno.readTextFileSync("tests/test-data/html-too-short-article.html"); + const result = await parseFromHtml(html, "abcd"); + assertEquals(result, null); +}); + +Deno.test("parseFromHtml - webpage with article but no source", async () => { + const html = Deno.readTextFileSync("tests/test-data/html-article-no-source.html"); + const result = await parseFromHtml(html); + assertEquals((result as Record).source, "somewhere.any"); +}); + +Deno.test("parseFromHtml - webpage with data-src in img tag", async () => { + const html = Deno.readTextFileSync("tests/test-data/html-article-with-data-src.html"); + const result = await parseFromHtml(html); + assertEquals((result as Record).content.includes(''), true); + assertEquals((result as Record).content.includes(''), true); +}); + +Deno.test("parseFromHtml - regular article", async () => { + const html = Deno.readTextFileSync("tests/test-data/regular-article.html"); + const result = await parseFromHtml(html, "https://somewhere.com/path/to/article"); + assertEquals((result as Record).title, "Article title here"); + assertEquals((result as Record).description, expDesc); + assertEquals((result as Record).content.includes(''), true); + assertEquals((result as Record).content.includes(''), true); +}); + +Deno.test("parseFromHtml - multi transforms", async () => { + addTransformations([ + { + patterns: [ + /http(s?):\/\/need-transform.tld\/*/, + ], + post: (document: Document) => { + document.querySelectorAll("a").forEach((node: Element) => { + const sHtml = node.innerHTML; + const link = node.getAttribute("href"); + node.parentNode!.replaceChild(document.createTextNode(`[link url="${link}"]${sHtml}[/link]`), node); + }); + return document; + }, + }, + { + patterns: [ + /http(s?):\/\/sw.re\/*/, + ], + post: (document: Document) => { + document.querySelectorAll("strong").forEach((node: Element) => { + const b = document.createElement("B"); + b.innerHTML = node.innerHTML; + node.parentNode!.replaceChild(b, node); + }); + return document; + }, + }, + ]); + const html = Deno.readTextFileSync("tests/test-data/vnn-article.html"); + const url = "https://need-transform.tld/path/to/article"; + const result = await parseFromHtml(html, url); + assertEquals((result as Record).title, "Article title here"); + assertEquals((result as Record).content.includes(''), false); + assertEquals((result as Record).content.includes('[link url="https://vnn.vn/dict/watermelon"]watermelon[/link]'), true); + assertEquals((result as Record).content.includes("in its own way"), true); + removeTransformations(); +}); diff --git a/tests/retrieve.test.ts b/tests/retrieve.test.ts new file mode 100644 index 0000000..7a5ddae --- /dev/null +++ b/tests/retrieve.test.ts @@ -0,0 +1,43 @@ +// tests/retrieve.test.ts + +import { assertEquals, assertRejects } from "@std/assert"; +import retrieve from "../src/utils/retrieve.ts"; +import { createMockFetcher } from "./helpers.ts"; + +Deno.test("retrieve - bad status code", async () => { + const url = "https://some.where/bad/page"; + const fetcher = createMockFetcher("Error 500", "text/plain", 500); + await assertRejects( + () => retrieve(url, fetcher), + Error, + "Request failed with error code 500", + ); +}); + +Deno.test("retrieve - good source", async () => { + const url = "https://some.where/good/page"; + const fetcher = createMockFetcher("
this is content
", "text/html"); + const buffer = await retrieve(url, fetcher); + const html = new TextDecoder().decode(buffer); + assertEquals(html, "
this is content
"); +}); + +Deno.test("retrieve - good source with \\r\\n", async () => { + const url = "https://some.where/good/page"; + const fetcher = createMockFetcher("\n\r\r\n\n
this is content
\n\r\r\n\n", "text/html"); + const buffer = await retrieve(url, fetcher); + const html = new TextDecoder().decode(buffer).trim(); + assertEquals(html, "
this is content
"); +}); + +Deno.test("retrieve - custom fetcher", async () => { + const url = "https://some.where/good/source-with-fetcher"; + const innerFetcher = createMockFetcher("
this is content
", "text/html"); + const myFetcher = (fetchUrl: string) => { + assertEquals(fetchUrl, url); + return innerFetcher(); + }; + const buffer = await retrieve(url, myFetcher); + const html = new TextDecoder().decode(buffer); + assertEquals(html, "
this is content
"); +}); diff --git a/test-data/article-with-classes-attributes.html b/tests/test-data/article-with-classes-attributes.html similarity index 100% rename from test-data/article-with-classes-attributes.html rename to tests/test-data/article-with-classes-attributes.html diff --git a/test-data/html-article-no-source.html b/tests/test-data/html-article-no-source.html similarity index 100% rename from test-data/html-article-no-source.html rename to tests/test-data/html-article-no-source.html diff --git a/test-data/html-article-with-data-src.html b/tests/test-data/html-article-with-data-src.html similarity index 100% rename from test-data/html-article-with-data-src.html rename to tests/test-data/html-article-with-data-src.html diff --git a/test-data/html-no-article.html b/tests/test-data/html-no-article.html similarity index 100% rename from test-data/html-no-article.html rename to tests/test-data/html-no-article.html diff --git a/test-data/html-no-link.html b/tests/test-data/html-no-link.html similarity index 100% rename from test-data/html-no-link.html rename to tests/test-data/html-no-link.html diff --git a/test-data/html-no-title.html b/tests/test-data/html-no-title.html similarity index 100% rename from test-data/html-no-title.html rename to tests/test-data/html-no-title.html diff --git a/test-data/html-too-short-article.html b/tests/test-data/html-too-short-article.html similarity index 100% rename from test-data/html-too-short-article.html rename to tests/test-data/html-too-short-article.html diff --git a/test-data/regular-article-date-itemprop.html b/tests/test-data/regular-article-date-itemprop.html similarity index 100% rename from test-data/regular-article-date-itemprop.html rename to tests/test-data/regular-article-date-itemprop.html diff --git a/test-data/regular-article-date-span.html b/tests/test-data/regular-article-date-span.html similarity index 100% rename from test-data/regular-article-date-span.html rename to tests/test-data/regular-article-date-span.html diff --git a/test-data/regular-article-date-time.html b/tests/test-data/regular-article-date-time.html similarity index 100% rename from test-data/regular-article-date-time.html rename to tests/test-data/regular-article-date-time.html diff --git a/test-data/regular-article-json-ld.html b/tests/test-data/regular-article-json-ld.html similarity index 100% rename from test-data/regular-article-json-ld.html rename to tests/test-data/regular-article-json-ld.html diff --git a/test-data/regular-article.html b/tests/test-data/regular-article.html similarity index 100% rename from test-data/regular-article.html rename to tests/test-data/regular-article.html diff --git a/test-data/vnn-article.html b/tests/test-data/vnn-article.html similarity index 100% rename from test-data/vnn-article.html rename to tests/test-data/vnn-article.html diff --git a/tests/transformation.test.ts b/tests/transformation.test.ts new file mode 100644 index 0000000..65dc18e --- /dev/null +++ b/tests/transformation.test.ts @@ -0,0 +1,191 @@ +// tests/transformation.test.ts + +import { assertEquals } from "@std/assert"; +import { + addTransformations, + removeTransformations, + getTransformations, + findTransformations, + execPreParser, + execPostParser, +} from "../src/utils/transformation.ts"; + +Deno.test("add one transformation object", () => { + const result = addTransformations({ + patterns: [ + /http(s?):\/\/([\w]+.)?def.tld\/*/, + ], + pre: (document: Document) => { + return document; + }, + post: (document: Document) => { + return document; + }, + }); + assertEquals(result, 1); +}); + +Deno.test("add multi transformation object", () => { + const result = addTransformations([ + { + patterns: [ + /http(s?):\/\/google.com\/*/, + /http(s?):\/\/goo.gl\/*/, + ], + }, + { + patterns: [ + /http(s?):\/\/goo.gl\/*/, + /http(s?):\/\/google.inc\/*/, + ], + }, + ]); + assertEquals(result, 2); +}); + +Deno.test("add transformation object without patterns", () => { + const result = addTransformations({ + pre: (document: Document) => { + return document; + }, + post: (document: Document) => { + return document; + }, + } as any); + assertEquals(result, 0); +}); + +Deno.test("add transformation object without valid patterns", () => { + const result = addTransformations({ + patterns: 123 as any, + pre: (document: Document) => { + return document; + }, + post: (document: Document) => { + return document; + }, + }); + assertEquals(result, 0); +}); + +Deno.test("get all transformations", () => { + const result = getTransformations(); + assertEquals(result.length, 3); + assertEquals(String(result[0].patterns[0]), String(/http(s?):\/\/([\w]+.)?def.tld\/*/)); +}); + +Deno.test("remove one transformation", () => { + addTransformations([ + { + patterns: [ + /http(s?):\/\/abc.com\/*/, + /http(s?):\/\/def.gl\/*/, + ], + }, + { + patterns: [ + /http(s?):\/\/hik.gl\/*/, + /http(s?):\/\/lmn.inc\/*/, + ], + }, + { + patterns: [ + /http(s?):\/\/opq.gl\/*/, + /http(s?):\/\/rst.inc\/*/, + ], + }, + ]); + const result = removeTransformations([ + /http(s?):\/\/goo.gl\/*/, + ]); + assertEquals(result, 2); +}); + +Deno.test("get all transformations again", () => { + const result = getTransformations(); + assertEquals(result.length, 4); + assertEquals(String(result[3].patterns[1]), String(/http(s?):\/\/rst.inc\/*/)); +}); + +Deno.test("find transformations", () => { + addTransformations([ + { + patterns: [ + /http(s?):\/\/def.gl\/*/, + /http(s?):\/\/uvw.inc\/*/, + ], + }, + ]); + const notFound = findTransformations([ + "https://goo.gl/docs/article.html", + ]); + assertEquals(notFound.length, 0); + + const foundOne = findTransformations([ + "https://lmn.inc/docs/article.html", + ]); + assertEquals(foundOne.length, 1); + + const foundTwo = findTransformations([ + "https://def.gl/docs/article.html", + ]); + assertEquals(foundTwo.length, 2); +}); + +Deno.test("execPreParser", () => { + addTransformations([ + { + patterns: [ + /http(s?):\/\/xyz.com\/*/, + ], + pre: (doc: Document) => { + doc.querySelectorAll(".adv").forEach((element: Element) => { + element.parentNode!.removeChild(element); + }); + return doc; + }, + }, + ]); + const html = ` +
+ hi user, this is an advertisement element +
free product now!
+
+ `; + const result = execPreParser(html, "https://xyz.com/article"); + assertEquals(result.includes("hi user, this is an advertisement element"), true); + assertEquals(result.includes('
free product now!
'), false); +}); + +Deno.test("execPostParser", () => { + addTransformations([ + { + patterns: [ + /http(s?):\/\/xyz.com\/*/, + ], + post: (doc: Document) => { + doc.querySelectorAll("b").forEach((element: Element) => { + const itag = doc.createElement("i"); + itag.innerHTML = element.innerHTML; + element.parentNode!.replaceChild(itag, element); + }); + return doc; + }, + }, + ]); + const html = ` +
+ hi user, +

Thank you for your feedback!

+
+ `; + const result = execPostParser(html, "https://xyz.com/article"); + assertEquals(result.includes("user"), true); + assertEquals(result.includes("user"), false); +}); + +Deno.test("remove all transformations", () => { + const result = removeTransformations(); + assertEquals(result, 7); + assertEquals(getTransformations().length, 0); +}); From 24a14ef463df8c25aca9f8c0c857dfd932bcbebf Mon Sep 17 00:00:00 2001 From: Dong Nguyen Date: Thu, 6 Aug 2026 18:33:07 +0700 Subject: [PATCH 4/6] v9.0.0-rc1 - Refactor HTML sainitizing --- README.md | 34 +++-- deno.json | 8 +- deno.lock | 76 +-------- mod.ts | 6 +- src/config.ts | 144 ++++++++++-------- src/main.ts | 12 +- src/utils/extractLdSchema.ts | 22 ++- src/utils/findDate.ts | 7 +- src/utils/html.ts | 87 ++++++++--- src/utils/linker.ts | 9 +- src/utils/parseFromHtml.ts | 55 ++++--- src/utils/retrieve.ts | 5 +- src/utils/similarity.ts | 18 ++- src/utils/transformation.ts | 38 +++-- tests/config.test.ts | 52 ++++--- tests/extractMetaData.test.ts | 22 ++- tests/extractWithReadability.test.ts | 9 +- tests/html.test.ts | 30 +++- tests/linker.test.ts | 101 +++++++++--- tests/main.test.ts | 52 ++++--- tests/parseFromHtml.test.ts | 76 +++++++-- tests/retrieve.test.ts | 10 +- .../article-with-classes-attributes.html | 30 ++-- tests/test-data/html-article-no-source.html | 18 +-- .../test-data/html-article-with-data-src.html | 14 +- tests/test-data/html-no-article.html | 14 +- tests/test-data/html-no-link.html | 6 +- tests/test-data/html-no-title.html | 11 +- tests/test-data/html-too-short-article.html | 18 +-- .../regular-article-date-itemprop.html | 27 ++-- .../test-data/regular-article-date-span.html | 26 ++-- .../test-data/regular-article-date-time.html | 27 ++-- tests/test-data/regular-article-json-ld.html | 65 ++++---- tests/test-data/regular-article.html | 30 ++-- tests/test-data/vnn-article.html | 24 ++- tests/transformation.test.ts | 28 +++- 36 files changed, 774 insertions(+), 437 deletions(-) diff --git a/README.md b/README.md index f6b6e5f..3db34ae 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ console.log(data); - [`addTransformations()`](#addtransformationstransformation--transformation) - [`removeTransformations()`](#removetransformationspatterns-regexp) - [Priority order](#priority-order) -- [`sanitize-html` options](#sanitize-html-options) +- [Content sanitization options](#content-sanitization-options) --- @@ -112,6 +112,9 @@ URL string or raw HTML content. | `descriptionTruncateLen` | `number` | `210` | Max characters for generated description | | `descriptionLengthThreshold` | `number` | `180` | Min characters to keep meta description | | `contentLengthThreshold` | `number` | `200` | Min characters for article content | +| `allowedTags` | `string[]` | *(semantic/content tags)* | HTML tags to keep in output | +| `allowedAttributes` | `Record` | *(src, href, alt, etc.)* | Per-tag attributes to keep | +| `allowedIframeDomains` | `string[]` | *(youtube, vimeo, etc.)* | Allowed domains for iframe src | ```ts const article = await extract(url, { @@ -322,25 +325,26 @@ pre_one -> pre_three -> extraction -> post_two -> post_four --- -### `sanitize-html` options +### Content sanitization options -Uses [sanitize-html](https://github.com/apostrophecms/sanitize-html) to clean extracted content. See the [default options](src/config.ts). +Extracted HTML is sanitized using a built-in DOM tree walker. Disallowed tags are removed (not escaped), and disallowed attributes are stripped. Configure via `parserOptions`: ```ts -import { getSanitizeHtmlOptions, setSanitizeHtmlOptions } from "jsr:@extractus/article-extractor"; - -// get current options (returns a clone) -const opts = getSanitizeHtmlOptions(); +import { extract } from "jsr:@extractus/article-extractor"; -// merge new options -setSanitizeHtmlOptions({ +// allow class attributes on and
+const article = await extract(url, { allowedAttributes: { - ...opts.allowedAttributes as Record, + a: ["href", "target", "title"], + img: ["src", "srcset", "alt", "title"], code: ["class"], + div: ["class"], }, }); ``` +To see the full defaults, refer to [`src/config.ts`](src/config.ts). + --- ## Development @@ -363,4 +367,14 @@ deno run -A ./scripts/build_npm.ts The MIT License (MIT) +## Support the project + +This project is maintained in my spare time. If you find it helpful, there are a few simple ways to support its continued development: + +* ⭐ Star this repository to help more people discover it. +* ☕ Buy me a coffee: https://paypal.me/ndaidong +* 🚀 Subscribe to the [Article Extractor service](https://rapidapi.com/pwshub-pwshub-default/api/article-extractor2) on RapidAPI. + +Every bit of support helps keep this project actively maintained. Thank you! â¤ī¸ + --- diff --git a/deno.json b/deno.json index d5692b7..fae3b00 100644 --- a/deno.json +++ b/deno.json @@ -1,6 +1,6 @@ { "name": "@extractus/article-extractor", - "version": "9.0.0-rc0", + "version": "9.0.0-rc1", "description": "To extract main article from given URL", "homepage": "https://github.com/extractus/article-extractor", "repository": { @@ -20,9 +20,7 @@ "@pwshub/bellajs": "jsr:@pwshub/bellajs@13.1.0", "@std/assert": "jsr:@std/assert@1.0.12", "@mozilla/readability": "npm:@mozilla/readability@^0.6.0", - "linkedom": "npm:linkedom@^0.18.13", - "sanitize-html": "npm:sanitize-html@^2.17.6", - "@types/sanitize-html": "npm:@types/sanitize-html@^2.16.1" + "linkedom": "npm:linkedom@^0.18.13" }, "compilerOptions": { "lib": ["dom", "dom.iterable", "dom.asynciterable", "deno.ns"] @@ -32,7 +30,7 @@ "exclude": ["npm/"], "rules": { "tags": ["recommended"], - "exclude": ["no-explicit-any"] + "exclude": ["no-explicit-any", "no-control-regex"] } }, "fmt": { diff --git a/deno.lock b/deno.lock index a211f92..8ea63eb 100644 --- a/deno.lock +++ b/deno.lock @@ -14,9 +14,7 @@ "jsr:@ts-morph/bootstrap@0.29": "0.29.0", "jsr:@ts-morph/common@0.29": "0.29.0", "npm:@mozilla/readability@0.6": "0.6.0", - "npm:@types/sanitize-html@^2.16.1": "2.16.1", - "npm:linkedom@~0.18.13": "0.18.13", - "npm:sanitize-html@^2.17.6": "2.17.6" + "npm:linkedom@~0.18.13": "0.18.13" }, "jsr": { "@david/code-block-writer@13.0.3": { @@ -78,12 +76,6 @@ "@mozilla/readability@0.6.0": { "integrity": "sha512-juG5VWh4qAivzTAeMzvY9xs9HY5rAcr2E4I7tiSSCokRFi7XIZCAu92ZkSTsIj1OPceCifL3cpfteP3pDT9/QQ==" }, - "@types/sanitize-html@2.16.1": { - "integrity": "sha512-n9wjs8bCOTyN/ynwD8s/nTcTreIHB1vf31vhLMGqUPNHaweKC4/fAl4Dj+hUlCTKYgm4P3k83fmiFfzkZ6sgMA==", - "dependencies": [ - "htmlparser2@10.1.0" - ] - }, "boolbase@2.0.0": { "integrity": "sha512-DkVaaQHymRhpYEYo9x1oo7Q7B0Y6KJUsjm3c9eTyFDby4MHLBTwZ6ZDWBel5zrYxj1WsZgC5oLpiz+93MluXeA==" }, @@ -103,12 +95,6 @@ "cssom@0.5.0": { "integrity": "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw==" }, - "dayjs@1.11.21": { - "integrity": "sha512-98IT+HOahAisibz/yjKbzuOBwYcjJ7BCLPzARyHiyEBmRz4fatF+KPJszEHXsGYjUG234aH/cOjW1wwTbKUZlA==" - }, - "deepmerge@4.3.1": { - "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==" - }, "dom-serializer@2.0.0": { "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", "dependencies": [ @@ -168,9 +154,6 @@ "entities@8.0.0": { "integrity": "sha512-zwfzJecQ/Uej6tusMqwAqU/6KL2XaB2VZ2Jg54Je6ahNBGNH6Ek6g3jjNCF0fG9EWQKGZNddNjU5F1ZQn/sBnA==" }, - "escape-string-regexp@4.0.0": { - "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==" - }, "html-escaper@3.0.3": { "integrity": "sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ==" }, @@ -183,73 +166,22 @@ "entities@7.0.1" ] }, - "htmlparser2@12.0.0": { - "integrity": "sha512-Tz7u1i95/g2x2jz81+x0FBVhBhY5aRTvD3tXXdFaljuNdzDLJ8UGNRrTcj2cgQvAg3iW/h77Fz15nLW0L0CrZw==", - "dependencies": [ - "domelementtype@3.0.0", - "domhandler@6.0.1", - "domutils@4.0.2", - "entities@8.0.0" - ] - }, - "is-plain-object@5.0.0": { - "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==" - }, - "launder@1.7.1": { - "integrity": "sha512-mU6WRz5EusL9ZZuiZ5SO4Y6C0P9PAUR9iwdb6bzj4KDihm28DiHFw+/yk9DBH4f+Pv1wuzQ4e2jV3oQ7mkIqvw==", - "dependencies": [ - "dayjs" - ] - }, "linkedom@0.18.13": { "integrity": "sha512-ES/o9qotMpzpN2MHs+Iq/JcVoOj8Fa5wiQYrTdFpvAnwXL0g66XHHUc9WUMk6nAlBtGsFQ24ne+SYnvnaQ2FSw==", "dependencies": [ "css-select", "cssom", "html-escaper", - "htmlparser2@10.1.0", + "htmlparser2", "uhyphen" ] }, - "nanoid@3.3.15": { - "integrity": "sha512-y7Wygv/7mEOvxTuEQDB8StXdMRBWf1kR/tlhAzBRUFkB2jfcLOAxO/SHmOO2zgz1pVgK29/kyupn059/bCHdjA==", - "bin": true - }, "nth-check@3.0.1": { "integrity": "sha512-GX0gsdbGVCgnRgbeGaubfjpBXyYRWOOCVeYh08bSQvDZqxz5ndXs1OTfAt/h36G1xvI94YIspsI0sVFqAV9+RQ==", "dependencies": [ "boolbase" ] }, - "parse-srcset@1.0.2": { - "integrity": "sha512-/2qh0lav6CmI15FzA3i/2Bzk2zCgQhGMkvhOhKNcBVQ1ldgpbfiNTVslmooUmWJcADi1f1kIeynbDRVzNlfR6Q==" - }, - "picocolors@1.1.1": { - "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==" - }, - "postcss@8.5.16": { - "integrity": "sha512-vuwillviilfKZsg0VGj5R/YwwcHx4SLsIOI/7K6mQkWx+l5cUHTjj5g0AasTBcyXsbfTgrwsUNmVUb5xVwyPwg==", - "dependencies": [ - "nanoid", - "picocolors", - "source-map-js" - ] - }, - "sanitize-html@2.17.6": { - "integrity": "sha512-M4bo9tfv1yfhQZZKkc6dL07ALrGJtfvNOuhX3hU9AVPR/uPQ+nKOJBqTYc7LfMQblTW04mtSWDJWEyLvygJsLA==", - "dependencies": [ - "deepmerge", - "escape-string-regexp", - "htmlparser2@12.0.0", - "is-plain-object", - "launder", - "parse-srcset", - "postcss" - ] - }, - "source-map-js@1.2.1": { - "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==" - }, "uhyphen@0.2.0": { "integrity": "sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA==" } @@ -260,9 +192,7 @@ "jsr:@pwshub/bellajs@13.1.0", "jsr:@std/assert@1.0.12", "npm:@mozilla/readability@0.6", - "npm:@types/sanitize-html@^2.16.1", - "npm:linkedom@~0.18.13", - "npm:sanitize-html@^2.17.6" + "npm:linkedom@~0.18.13" ] } } diff --git a/mod.ts b/mod.ts index 52d0e7b..6842e09 100644 --- a/mod.ts +++ b/mod.ts @@ -1,14 +1,12 @@ export { + addTransformations, extract, extractFromHtml, - addTransformations, removeTransformations, - setSanitizeHtmlOptions, - getSanitizeHtmlOptions, } from "./src/main.ts"; export type { ArticleData, - ParserOptions, Fetcher, + ParserOptions, Transformation, } from "./src/main.ts"; diff --git a/src/config.ts b/src/config.ts index 33aa9b4..1504747 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,65 +1,89 @@ // config.ts -import { clone } from "@pwshub/bellajs"; +export const defaultAllowedTags: string[] = [ + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "u", + "b", + "i", + "em", + "strong", + "small", + "sup", + "sub", + "div", + "span", + "p", + "article", + "blockquote", + "section", + "details", + "summary", + "pre", + "code", + "ul", + "ol", + "li", + "dd", + "dl", + "table", + "th", + "tr", + "td", + "thead", + "tbody", + "tfoot", + "fieldset", + "legend", + "figure", + "figcaption", + "img", + "picture", + "video", + "audio", + "source", + "iframe", + "progress", + "br", + "hr", + "label", + "abbr", + "a", + "svg", +]; -const sanitizeHtmlOptions: Record = { - allowedTags: [ - "h1", "h2", "h3", "h4", "h5", "h6", - "u", "b", "i", "em", "strong", "small", "sup", "sub", - "div", "span", "p", "article", "blockquote", "section", - "details", "summary", - "pre", "code", - "ul", "ol", "li", "dd", "dl", - "table", "th", "tr", "td", "thead", "tbody", "tfoot", - "fieldset", "legend", - "figure", "figcaption", "img", "picture", - "video", "audio", "source", - "iframe", - "progress", - "br", "p", "hr", - "label", - "abbr", - "a", - "svg", - ], - allowedAttributes: { - h1: ["id"], - h2: ["id"], - h3: ["id"], - h4: ["id"], - h5: ["id"], - h6: ["id"], - a: ["href", "target", "title"], - abbr: ["title"], - progress: ["value", "max"], - img: ["src", "srcset", "alt", "title"], - picture: ["media", "srcset"], - video: ["controls", "width", "height", "autoplay", "muted", "loop", "src"], - audio: ["controls", "width", "height", "autoplay", "muted", "loop", "src"], - source: ["src", "srcset", "data-srcset", "type", "media", "sizes"], - iframe: ["src", "frameborder", "height", "width", "scrolling", "allow"], - svg: ["width", "height"], - }, - allowedIframeDomains: [ - "youtube.com", "vimeo.com", "odysee.com", - "soundcloud.com", "audius.co", - "github.com", "codepen.com", - "twitter.com", "facebook.com", "instagram.com", - ], - disallowedTagsMode: "discard", - allowVulnerableTags: false, - parseStyleAttributes: false, - enforceHtmlBoundary: false, +export const defaultAllowedAttributes: Record = { + h1: ["id"], + h2: ["id"], + h3: ["id"], + h4: ["id"], + h5: ["id"], + h6: ["id"], + a: ["href", "target", "title"], + abbr: ["title"], + progress: ["value", "max"], + img: ["src", "srcset", "alt", "title"], + picture: ["media", "srcset"], + video: ["controls", "width", "height", "autoplay", "muted", "loop", "src"], + audio: ["controls", "width", "height", "autoplay", "muted", "loop", "src"], + source: ["src", "srcset", "data-srcset", "type", "media", "sizes"], + iframe: ["src", "frameborder", "height", "width", "scrolling", "allow"], + svg: ["width", "height"], }; -/** Get a clone of the current sanitize-html options. */ -export const getSanitizeHtmlOptions = (): Record => { - return clone(sanitizeHtmlOptions); -}; - -/** Update sanitize-html options by merging with the current ones. */ -export const setSanitizeHtmlOptions = (opts: Record = {}): void => { - Object.keys(opts).forEach((key) => { - (sanitizeHtmlOptions as Record)[key] = clone(opts[key]); - }); -}; +export const defaultAllowedIframeDomains: string[] = [ + "youtube.com", + "vimeo.com", + "odysee.com", + "soundcloud.com", + "audius.co", + "github.com", + "codepen.com", + "twitter.com", + "facebook.com", + "instagram.com", +]; diff --git a/src/main.ts b/src/main.ts index e1c0236..06e3d34 100644 --- a/src/main.ts +++ b/src/main.ts @@ -30,6 +30,12 @@ export interface ParserOptions { descriptionLengthThreshold?: number; /** Min characters required for article content. Default: 200 */ contentLengthThreshold?: number; + /** Allowed HTML tags in output. Default: list of semantic/content tags. */ + allowedTags?: string[]; + /** Per-tag allowed attributes. Default: src, href, alt, etc. */ + allowedAttributes?: Record; + /** Allowed domains for iframe src. Default: youtube, vimeo, etc. */ + allowedIframeDomains?: string[]; } /** Extracted article data structure. */ @@ -108,5 +114,7 @@ export const extractFromHtml = async ( return parseFromHtml(html, url || "", parserOptions); }; -export { addTransformations, removeTransformations } from "./utils/transformation.ts"; -export { setSanitizeHtmlOptions, getSanitizeHtmlOptions } from "./config.ts"; +export { + addTransformations, + removeTransformations, +} from "./utils/transformation.ts"; diff --git a/src/utils/extractLdSchema.ts b/src/utils/extractLdSchema.ts index 7d6ed1b..179b4f9 100644 --- a/src/utils/extractLdSchema.ts +++ b/src/utils/extractLdSchema.ts @@ -49,13 +49,21 @@ const parseJson = (text: string): Record => { const isAllowedLdJsonType = (ldJson: Record): boolean => { const rootLdJsonType = ldJson["@type"] || ""; - const arr = isArray(rootLdJsonType) ? rootLdJsonType as string[] : [rootLdJsonType as string]; + const arr = isArray(rootLdJsonType) + ? rootLdJsonType as string[] + : [rootLdJsonType as string]; const ldJsonTypes = arr.filter((x) => !!x); - return ldJsonTypes.length > 0 && ldJsonTypes.some((x) => typeSchemas.includes(x.toLowerCase())); + return ldJsonTypes.length > 0 && + ldJsonTypes.some((x) => typeSchemas.includes(x.toLowerCase())); }; -export default (document: Document, entry: Record): Record => { - const ldSchemas = document.querySelectorAll('script[type="application/ld+json"]'); +export default ( + document: Document, + entry: Record, +): Record => { + const ldSchemas = document.querySelectorAll( + 'script[type="application/ld+json"]', + ); ldSchemas.forEach((ldSchema: Element) => { const ldJson = parseJson(ldSchema.textContent.replace(/[\n\r\t]/g, "")); if (ldJson && isAllowedLdJsonType(ldJson)) { @@ -65,7 +73,11 @@ export default (document: Document, entry: Record): Record)?.name || "" : keyValue; + const val = isArray(keyValue) + ? (keyValue as unknown[])[0] + : isObject(keyValue) + ? (keyValue as Record)?.name || "" + : keyValue; if (isString(val) && val !== "") { entry[key] = (val as string).trim(); } diff --git a/src/utils/findDate.ts b/src/utils/findDate.ts index d6bcca8..b5b361c 100644 --- a/src/utils/findDate.ts +++ b/src/utils/findDate.ts @@ -32,9 +32,12 @@ export default function findDate(doc: Document): string { return ""; }; - const priorityElements = doc.querySelectorAll("time, [datetime], [itemprop~=datePublished], [itemprop~=dateCreated]"); + const priorityElements = doc.querySelectorAll( + "time, [datetime], [itemprop~=datePublished], [itemprop~=dateCreated]", + ); for (const el of Array.from(priorityElements)) { - const date = el.getAttribute("datetime") || el.getAttribute("content") || findDateInElement(el); + const date = el.getAttribute("datetime") || el.getAttribute("content") || + findDateInElement(el); if (date) return date; } diff --git a/src/utils/html.ts b/src/utils/html.ts index 0b5b542..beb5a4e 100644 --- a/src/utils/html.ts +++ b/src/utils/html.ts @@ -1,21 +1,15 @@ // utils -> html.ts import { DOMParser } from "linkedom"; -import sanitize from "sanitize-html"; -import { pipe } from "@pwshub/bellajs"; -import { getSanitizeHtmlOptions } from "../config.ts"; +export interface SanitizeOptions { + allowedTags: string[]; + allowedAttributes: Record; + allowedIframeDomains: string[]; +} -export const purify = (html: string): string => { - return sanitize(html, { - allowedTags: false, - allowedAttributes: false, - allowVulnerableTags: true, - }); -}; - -// deno-lint-ignore no-control-regex -const WS_REGEXP = /^[\s\f\n\r\t\u1680\u180e\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000\ufeff\x09\x0a\x0b\x0c\x0d\x20\xa0]+$/; +const WS_REGEXP = + /^[\s\f\n\r\t\u1680\u180e\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000\ufeff\x09\x0a\x0b\x0c\x0d\x20\xa0]+$/; const stripMultiLinebreaks = (str: string): string => { return str.replace(/(\r\n|\n|\u2424){2,}/g, "\n").split("\n").map((line) => { @@ -35,19 +29,70 @@ export const getCharset = (html: string): string => { let charset = m ? m.getAttribute("charset") : ""; if (!charset) { const h = doc.querySelector('meta[http-equiv="content-type"]') || null; - charset = h ? h.getAttribute("content")?.split(";")[1]?.replace("charset=", "")?.trim() : ""; + charset = h + ? h.getAttribute("content")?.split(";")[1]?.replace("charset=", "") + ?.trim() + : ""; } return charset?.toLowerCase() || "utf8"; }; -export const cleanify = (inputHtml: string): string => { +export const cleanify = ( + inputHtml: string, + options: SanitizeOptions, +): string => { + const { allowedTags, allowedAttributes, allowedIframeDomains } = options; + const tagSet = new Set(allowedTags.map((t) => t.toLowerCase())); const doc = new DOMParser().parseFromString(inputHtml, "text/html"); - const html = doc.documentElement.innerHTML; - return pipe( - (input: string) => sanitize(input, getSanitizeHtmlOptions() as sanitize.IOptions), - (input: string) => stripMultiLinebreaks(input), - (input: string) => stripMultispaces(input), - )(html); + + const sanitize = (node: ChildNode) => { + if (node.nodeType !== 1) return; + const el = node as Element; + const tag = el.tagName.toLowerCase(); + + if (!tagSet.has(tag)) { + el.parentNode?.removeChild(el); + return; + } + + if (tag === "iframe") { + const src = el.getAttribute("src") || ""; + try { + const hostname = new URL(src).hostname.replace(/^www\./, ""); + const allowed = allowedIframeDomains.some((d) => + hostname === d || hostname.endsWith("." + d) + ); + if (!allowed) { + el.parentNode?.removeChild(el); + return; + } + } catch { + el.parentNode?.removeChild(el); + return; + } + } + + const allowed = allowedAttributes[tag]; + const allowedSet = allowed ? new Set(allowed) : null; + for (let i = el.attributes.length - 1; i >= 0; i--) { + const attr = el.attributes[i]; + if (!allowedSet || !allowedSet.has(attr.name)) { + el.removeAttribute(attr.name); + } + } + + const children = Array.from(node.childNodes); + for (const child of children) { + sanitize(child); + } + }; + + const children = Array.from(doc.documentElement.childNodes); + for (const child of children) { + sanitize(child); + } + + return stripMultiLinebreaks(stripMultispaces(doc.documentElement.innerHTML)); }; export const countImages = (html: string): number => { diff --git a/src/utils/linker.ts b/src/utils/linker.ts index 79fc2c8..c9f912b 100644 --- a/src/utils/linker.ts +++ b/src/utils/linker.ts @@ -13,7 +13,10 @@ export const isValid = (url: string = ""): boolean => { } }; -export const chooseBestUrl = (candidates: string[] = [], title = ""): string => { +export const chooseBestUrl = ( + candidates: string[] = [], + title = "", +): string => { const ranking = findBestMatch(title, candidates); return ranking.bestMatch.target; }; @@ -126,7 +129,9 @@ export const normalize = (html: string, url: string): string => { } }); - return Array.from(doc.childNodes).map((element) => (element as Element).outerHTML).join(""); + return Array.from(doc.childNodes).map((element) => + (element as Element).outerHTML + ).join(""); }; export const getDomain = (url: string): string => { diff --git a/src/utils/parseFromHtml.ts b/src/utils/parseFromHtml.ts index f90bde2..038c017 100644 --- a/src/utils/parseFromHtml.ts +++ b/src/utils/parseFromHtml.ts @@ -1,26 +1,22 @@ // utils -> parseFromHtml.ts import { + getTTR, + pipe, stripTags, truncateByChar, unique, - pipe, - getTTR, } from "@pwshub/bellajs"; -import { - purify, - cleanify, - countImages, -} from "./html.ts"; +import { cleanify, countImages } from "./html.ts"; import { - isValid as isValidUrl, - purify as purifyUrl, absolutify as absolutifyUrl, - normalize as normalizeUrls, chooseBestUrl, getDomain, + isValid as isValidUrl, + normalize as normalizeUrls, + purify as purifyUrl, } from "./linker.ts"; import extractMetaData from "./extractMetaData.ts"; @@ -29,20 +25,36 @@ import extractWithReadability, { extractTitleWithReadability, } from "./extractWithReadability.ts"; -import { execPreParser, execPostParser } from "./transformation.ts"; +import { execPostParser, execPreParser } from "./transformation.ts"; + +import { + defaultAllowedAttributes, + defaultAllowedIframeDomains, + defaultAllowedTags, +} from "../config.ts"; import type { ArticleData, ParserOptions } from "../main.ts"; -const summarize = ({ desc, text, threshold, maxlen }: { desc: string; text: string; threshold: number; maxlen: number }): string => { +const summarize = ( + { desc, text, threshold, maxlen }: { + desc: string; + text: string; + threshold: number; + maxlen: number; + }, +): string => { return desc.length > threshold ? desc : truncateByChar(text, maxlen).replace(/\n/g, " "); }; // deno-lint-ignore require-await -export default async (inputHtml: string, inputUrl = "", parserOptions: ParserOptions = {}): Promise => { - const pureHtml = purify(inputHtml); - const meta = extractMetaData(pureHtml); +export default async ( + inputHtml: string, + inputUrl = "", + parserOptions: ParserOptions = {}, +): Promise => { + const meta = extractMetaData(inputHtml); let title = meta.title; @@ -64,10 +76,13 @@ export default async (inputHtml: string, inputUrl = "", parserOptions: ParserOpt descriptionTruncateLen = 210, descriptionLengthThreshold = 180, contentLengthThreshold = 200, + allowedTags = defaultAllowedTags, + allowedAttributes = defaultAllowedAttributes, + allowedIframeDomains = defaultAllowedIframeDomains, } = parserOptions; if (!title) { - title = extractTitleWithReadability(pureHtml) || ""; + title = extractTitleWithReadability(inputHtml) || ""; } if (!title) { return null; @@ -99,7 +114,13 @@ export default async (inputHtml: string, inputUrl = "", parserOptions: ParserOpt return input ? execPostParser(input, links) : null; }, (input: string | null) => { - return input ? cleanify(input) : null; + return input + ? cleanify(input, { + allowedTags, + allowedAttributes, + allowedIframeDomains, + }) + : null; }, ); diff --git a/src/utils/retrieve.ts b/src/utils/retrieve.ts index ca894f4..6545be5 100644 --- a/src/utils/retrieve.ts +++ b/src/utils/retrieve.ts @@ -1,6 +1,9 @@ // utils -> retrieve.ts -export default async (url: string, fetcher: (url: string) => Promise): Promise => { +export default async ( + url: string, + fetcher: (url: string) => Promise, +): Promise => { const res = await fetcher(url); const status = res.status; diff --git a/src/utils/similarity.ts b/src/utils/similarity.ts index be6a102..e25ff11 100644 --- a/src/utils/similarity.ts +++ b/src/utils/similarity.ts @@ -1,10 +1,6 @@ // utils -> similarity.ts -import { - isString, - compareTwoStrings, - isArray, -} from "@pwshub/bellajs"; +import { compareTwoStrings, isArray, isString } from "@pwshub/bellajs"; interface MatchResult { target: string; @@ -19,12 +15,18 @@ interface BestMatchResult { const areArgsValid = (mainString: unknown, targetStrings: unknown): boolean => { return isString(mainString) && isArray(targetStrings) && - (targetStrings as unknown[]).length > 0 && (targetStrings as string[]).every((s) => isString(s)); + (targetStrings as unknown[]).length > 0 && + (targetStrings as string[]).every((s) => isString(s)); }; -export const findBestMatch = (mainString: string, targetStrings: string[]): BestMatchResult => { +export const findBestMatch = ( + mainString: string, + targetStrings: string[], +): BestMatchResult => { if (!areArgsValid(mainString, targetStrings)) { - throw new Error("Bad arguments: First argument should be a string, second should be an array of strings"); + throw new Error( + "Bad arguments: First argument should be a string, second should be an array of strings", + ); } const ratings: MatchResult[] = []; diff --git a/src/utils/transformation.ts b/src/utils/transformation.ts index 2c897c6..8a1fa8d 100644 --- a/src/utils/transformation.ts +++ b/src/utils/transformation.ts @@ -21,9 +21,13 @@ const add = (tn: Transformation): number => { }; /** Register one or more transformations for per-site HTML processing. */ -export const addTransformations = (tfms: Transformation | Transformation[]): number => { +export const addTransformations = ( + tfms: Transformation | Transformation[], +): number => { if (isArray(tfms)) { - return (tfms as Transformation[]).map((tfm) => add(tfm)).filter((result) => result === 1).length; + return (tfms as Transformation[]).map((tfm) => add(tfm)).filter((result) => + result === 1 + ).length; } return add(tfms as Transformation); }; @@ -53,7 +57,9 @@ export const getTransformations = (): Transformation[] => { return [...transformations]; }; -export const findTransformations = (links: string | string[]): Transformation[] => { +export const findTransformations = ( + links: string | string[], +): Transformation[] => { const urls = !isArray(links) ? [links] : links; const tfms: Transformation[] = []; for (const transformation of transformations) { @@ -70,14 +76,28 @@ export const findTransformations = (links: string | string[]): Transformation[] return tfms; }; -export const execPreParser = (html: string, links: string | string[]): string => { +export const execPreParser = ( + html: string, + links: string | string[], +): string => { const doc = new DOMParser().parseFromString(html, "text/html"); - findTransformations(links).map((tfm) => tfm.pre).filter((fn) => isFunction(fn)).forEach((fn) => (fn as (doc: Document) => Document)(doc as any)); - return Array.from(doc.childNodes).map((it) => (it as Element).outerHTML).join(""); + findTransformations(links).map((tfm) => tfm.pre).filter((fn) => + isFunction(fn) + ).forEach((fn) => (fn as (doc: Document) => Document)(doc as any)); + return Array.from(doc.childNodes).map((it) => (it as Element).outerHTML).join( + "", + ); }; -export const execPostParser = (html: string, links: string | string[]): string => { +export const execPostParser = ( + html: string, + links: string | string[], +): string => { const doc = new DOMParser().parseFromString(html, "text/html"); - findTransformations(links).map((tfm) => tfm.post).filter((fn) => isFunction(fn)).forEach((fn) => (fn as (doc: Document) => Document)(doc as any)); - return Array.from(doc.childNodes).map((it) => (it as Element).outerHTML).join(""); + findTransformations(links).map((tfm) => tfm.post).filter((fn) => + isFunction(fn) + ).forEach((fn) => (fn as (doc: Document) => Document)(doc as any)); + return Array.from(doc.childNodes).map((it) => (it as Element).outerHTML).join( + "", + ); }; diff --git a/tests/config.test.ts b/tests/config.test.ts index d260d9b..c006e2c 100644 --- a/tests/config.test.ts +++ b/tests/config.test.ts @@ -1,31 +1,35 @@ // tests/config.test.ts import { assertEquals } from "@std/assert"; -import { setSanitizeHtmlOptions, getSanitizeHtmlOptions } from "../src/config.ts"; +import { + defaultAllowedAttributes, + defaultAllowedIframeDomains, + defaultAllowedTags, +} from "../src/config.ts"; -Deno.test("setSanitizeHtmlOptions/getSanitizeHtmlOptions", () => { - setSanitizeHtmlOptions({ - allowedTags: ["div", "span"], - allowedAttributes: { - a: ["href", "title"], - }, - }); - - const actual = getSanitizeHtmlOptions(); - const actualAllowedAttributes = actual.allowedAttributes as Record; - const expectedAllowedAttributes = { - a: ["href", "title"], - }; - - assertEquals(actualAllowedAttributes, expectedAllowedAttributes); - - const actualAllowedTags = actual.allowedTags as string[]; - const expectedAllowedTags = ["div", "span"]; - assertEquals(actualAllowedTags, expectedAllowedTags); +Deno.test("defaultAllowedTags - contains expected tags", () => { + assertEquals(defaultAllowedTags.includes("a"), true); + assertEquals(defaultAllowedTags.includes("img"), true); + assertEquals(defaultAllowedTags.includes("p"), true); + assertEquals(defaultAllowedTags.includes("script"), false); + assertEquals(defaultAllowedTags.includes("style"), false); +}); - setSanitizeHtmlOptions({ - allowedTags: [], - }); +Deno.test("defaultAllowedAttributes - contains expected attributes", () => { + assertEquals(defaultAllowedAttributes.a, ["href", "target", "title"]); + assertEquals(defaultAllowedAttributes.img, ["src", "srcset", "alt", "title"]); + assertEquals(defaultAllowedAttributes.iframe, [ + "src", + "frameborder", + "height", + "width", + "scrolling", + "allow", + ]); +}); - assertEquals(getSanitizeHtmlOptions().allowedTags, []); +Deno.test("defaultAllowedIframeDomains - contains expected domains", () => { + assertEquals(defaultAllowedIframeDomains.includes("youtube.com"), true); + assertEquals(defaultAllowedIframeDomains.includes("vimeo.com"), true); + assertEquals(defaultAllowedIframeDomains.includes("evil.com"), false); }); diff --git a/tests/extractMetaData.test.ts b/tests/extractMetaData.test.ts index 46a1b20..38a107d 100644 --- a/tests/extractMetaData.test.ts +++ b/tests/extractMetaData.test.ts @@ -1,10 +1,12 @@ // tests/extractMetaData.test.ts import { assertEquals } from "@std/assert"; -import { isObject, hasProperty } from "@pwshub/bellajs"; +import { hasProperty, isObject } from "@pwshub/bellajs"; import extractMetaData from "../src/utils/extractMetaData.ts"; -const keys = "url shortlink amphtml canonical title description image author source published favicon type".split(" "); +const keys = + "url shortlink amphtml canonical title description image author source published favicon type" + .split(" "); function isDateString(date: unknown): boolean { if (typeof date !== "string") return false; @@ -22,7 +24,9 @@ Deno.test("extractMetaData - good content", () => { }); Deno.test("extractMetaData - json ld schema content", () => { - const html = Deno.readTextFileSync("tests/test-data/regular-article-json-ld.html"); + const html = Deno.readTextFileSync( + "tests/test-data/regular-article-json-ld.html", + ); const result = extractMetaData(html); assertEquals(isObject(result), true); keys.forEach((k) => { @@ -52,9 +56,15 @@ Deno.test("extractMetaData - json ld schema fills empty meta fields", () => { }); Deno.test("extractMetaData - find date", () => { - const html1 = Deno.readTextFileSync("tests/test-data/regular-article-date-time.html"); - const html2 = Deno.readTextFileSync("tests/test-data/regular-article-date-itemprop.html"); - const html3 = Deno.readTextFileSync("tests/test-data/regular-article-date-span.html"); + const html1 = Deno.readTextFileSync( + "tests/test-data/regular-article-date-time.html", + ); + const html2 = Deno.readTextFileSync( + "tests/test-data/regular-article-date-itemprop.html", + ); + const html3 = Deno.readTextFileSync( + "tests/test-data/regular-article-date-span.html", + ); const result1 = extractMetaData(html1); const result2 = extractMetaData(html2); const result3 = extractMetaData(html3); diff --git a/tests/extractWithReadability.test.ts b/tests/extractWithReadability.test.ts index c3f90eb..46da44d 100644 --- a/tests/extractWithReadability.test.ts +++ b/tests/extractWithReadability.test.ts @@ -2,14 +2,19 @@ import { assertEquals } from "@std/assert"; import { isString } from "@pwshub/bellajs"; -import extractWithReadability, { extractTitleWithReadability } from "../src/utils/extractWithReadability.ts"; +import extractWithReadability, { + extractTitleWithReadability, +} from "../src/utils/extractWithReadability.ts"; Deno.test("extractWithReadability - good html content", () => { const html = Deno.readTextFileSync("tests/test-data/regular-article.html"); const result = extractWithReadability(html, "https://foo.bar"); assertEquals(isString(result), true); assertEquals((result as string).length > 200, true); - assertEquals((result as string).includes(''), true); + assertEquals( + (result as string).includes(''), + true, + ); }); Deno.test("extractWithReadability - bad html content", () => { diff --git a/tests/html.test.ts b/tests/html.test.ts index bc03e96..bcdbf49 100644 --- a/tests/html.test.ts +++ b/tests/html.test.ts @@ -3,13 +3,33 @@ import { assertEquals } from "@std/assert"; import { isString } from "@pwshub/bellajs"; import { cleanify } from "../src/utils/html.ts"; +import { + defaultAllowedAttributes, + defaultAllowedIframeDomains, + defaultAllowedTags, +} from "../src/config.ts"; + +const defaults = { + allowedTags: defaultAllowedTags, + allowedAttributes: defaultAllowedAttributes, + allowedIframeDomains: defaultAllowedIframeDomains, +}; Deno.test("cleanify - removes unwanted elements/attributes", () => { const html = Deno.readTextFileSync("tests/test-data/regular-article.html"); - assertEquals(html.includes("
4746 Kelly Drive, West Virginia
"), true); - assertEquals(html.includes(''), true); - const result = cleanify(html); + assertEquals( + html.includes("
4746 Kelly Drive, West Virginia
"), + true, + ); + assertEquals( + html.includes(''), + true, + ); + const result = cleanify(html, defaults); assertEquals(isString(result), true); - assertEquals(result.includes("
4746 Kelly Drive, West Virginia
"), false); - assertEquals(result.includes(''), false); + assertEquals( + result.includes("
4746 Kelly Drive, West Virginia
"), + false, + ); + assertEquals(result.includes('style="border: solid 1px #000"'), false); }); diff --git a/tests/linker.test.ts b/tests/linker.test.ts index e92a2bb..12f7f2a 100644 --- a/tests/linker.test.ts +++ b/tests/linker.test.ts @@ -3,17 +3,22 @@ import { assertEquals } from "@std/assert"; import { isString } from "@pwshub/bellajs"; import { + absolutify as absolutifyUrl, chooseBestUrl, isValid as isValidUrl, - purify as purifyUrl, normalize as normalizeUrls, - absolutify as absolutifyUrl, + purify as purifyUrl, } from "../src/utils/linker.ts"; Deno.test("isValidUrl - valid URLs", () => { assertEquals(isValidUrl("https://www.23hq.com"), true); assertEquals(isValidUrl("https://secure.actblue.com"), true); - assertEquals(isValidUrl("https://docs.microsoft.com/en-us/azure/iot-edge/quickstart?view=iotedge-2018-06"), true); + assertEquals( + isValidUrl( + "https://docs.microsoft.com/en-us/azure/iot-edge/quickstart?view=iotedge-2018-06", + ), + true, + ); assertEquals(isValidUrl("http://192.168.1.199:8081/example/page"), true); }); @@ -29,30 +34,92 @@ Deno.test("normalizeUrls - adds absolute URLs and target=_blank", () => { const html = Deno.readTextFileSync("tests/test-data/regular-article.html"); const result = normalizeUrls(html, bestUrl); assertEquals(isString(result), true); - assertEquals(result.includes('
watermelon'), false); - assertEquals(result.includes('watermelon'), true); - assertEquals(result.includes('rational peach'), false); - assertEquals(result.includes('rational peach'), true); + assertEquals( + result.includes('watermelon'), + false, + ); + assertEquals( + result.includes( + 'watermelon', + ), + true, + ); + assertEquals( + result.includes( + 'rational peach', + ), + false, + ); + assertEquals( + result.includes( + 'rational peach', + ), + true, + ); }); Deno.test("purifyUrl - various cases", () => { assertEquals(purifyUrl(""), null); assertEquals(purifyUrl({} as unknown as string), null); - assertEquals(purifyUrl("https://some.where/article/abc-xyz"), "https://some.where/article/abc-xyz"); - assertEquals(purifyUrl("https://some.where/article/abc-xyz#name,bob"), "https://some.where/article/abc-xyz"); - assertEquals(purifyUrl("https://some.where/article/abc-xyz?utm_source=news4&utm_medium=email&utm_campaign=spring-summer"), "https://some.where/article/abc-xyz"); - assertEquals(purifyUrl("https://some.where/article/abc-xyz?q=3&utm_source=news4&utm_medium=email&utm_campaign=spring-summer"), "https://some.where/article/abc-xyz?q=3"); - assertEquals(purifyUrl("https://some.where/article/abc-xyz?pk_source=news4&pk_medium=email&pk_campaign=spring-summer"), "https://some.where/article/abc-xyz"); - assertEquals(purifyUrl("https://some.where/article/abc-xyz?q=3&pk_source=news4&pk_medium=email&pk_campaign=spring-summer"), "https://some.where/article/abc-xyz?q=3"); + assertEquals( + purifyUrl("https://some.where/article/abc-xyz"), + "https://some.where/article/abc-xyz", + ); + assertEquals( + purifyUrl("https://some.where/article/abc-xyz#name,bob"), + "https://some.where/article/abc-xyz", + ); + assertEquals( + purifyUrl( + "https://some.where/article/abc-xyz?utm_source=news4&utm_medium=email&utm_campaign=spring-summer", + ), + "https://some.where/article/abc-xyz", + ); + assertEquals( + purifyUrl( + "https://some.where/article/abc-xyz?q=3&utm_source=news4&utm_medium=email&utm_campaign=spring-summer", + ), + "https://some.where/article/abc-xyz?q=3", + ); + assertEquals( + purifyUrl( + "https://some.where/article/abc-xyz?pk_source=news4&pk_medium=email&pk_campaign=spring-summer", + ), + "https://some.where/article/abc-xyz", + ); + assertEquals( + purifyUrl( + "https://some.where/article/abc-xyz?q=3&pk_source=news4&pk_medium=email&pk_campaign=spring-summer", + ), + "https://some.where/article/abc-xyz?q=3", + ); }); Deno.test("absolutifyUrl - various cases", () => { assertEquals(absolutifyUrl("", ""), ""); assertEquals(absolutifyUrl("", {} as unknown as string), ""); - assertEquals(absolutifyUrl("https://some.where/article/abc-xyz", "category/page.html"), "https://some.where/article/category/page.html"); - assertEquals(absolutifyUrl("https://some.where/article/abc-xyz", "../category/page.html"), "https://some.where/category/page.html"); - assertEquals(absolutifyUrl("https://some.where/blog/authors/article/abc-xyz", "/category/page.html"), "https://some.where/category/page.html"); - assertEquals(absolutifyUrl("https://some.where/article/abc-xyz"), "https://some.where/article/abc-xyz"); + assertEquals( + absolutifyUrl("https://some.where/article/abc-xyz", "category/page.html"), + "https://some.where/article/category/page.html", + ); + assertEquals( + absolutifyUrl( + "https://some.where/article/abc-xyz", + "../category/page.html", + ), + "https://some.where/category/page.html", + ); + assertEquals( + absolutifyUrl( + "https://some.where/blog/authors/article/abc-xyz", + "/category/page.html", + ), + "https://some.where/category/page.html", + ); + assertEquals( + absolutifyUrl("https://some.where/article/abc-xyz"), + "https://some.where/article/abc-xyz", + ); }); Deno.test("chooseBestUrl - actual case", () => { diff --git a/tests/main.test.ts b/tests/main.test.ts index 234ff62..5c5fb65 100644 --- a/tests/main.test.ts +++ b/tests/main.test.ts @@ -1,11 +1,12 @@ // tests/main.test.ts import { assertEquals, assertRejects } from "@std/assert"; +import { extract } from "../src/main.ts"; import { - extract, - getSanitizeHtmlOptions, - setSanitizeHtmlOptions, -} from "../src/main.ts"; + defaultAllowedAttributes, + defaultAllowedIframeDomains, + defaultAllowedTags, +} from "../src/config.ts"; import { createMockFetcher } from "./helpers.ts"; Deno.test("extract - non-string input", async () => { @@ -41,7 +42,11 @@ Deno.test("extract - regular article from url", async () => { const html = Deno.readTextFileSync("tests/test-data/regular-article.html"); const fetcher = createMockFetcher(html, "text/html"); - const result = await extract("https://somewhere.com/path/to/article", {}, fetcher); + const result = await extract( + "https://somewhere.com/path/to/article", + {}, + fetcher, + ); assertEquals((result as Record).title, "Article title here"); assertEquals((result as Record).description, expDesc); }); @@ -49,13 +54,21 @@ Deno.test("extract - regular article from url", async () => { Deno.test("extract - no article from url", async () => { const html = Deno.readTextFileSync("tests/test-data/html-no-article.html"); const fetcher = createMockFetcher(html, "text/html"); - const result = await extract("https://somewhere.com/path/to/no/article", {}, fetcher); + const result = await extract( + "https://somewhere.com/path/to/no/article", + {}, + fetcher, + ); assertEquals(result, null); }); Deno.test("extract - empty content from url", async () => { const fetcher = createMockFetcher("", "text/html"); - const result = await extract("https://somewhere.com/path/to/no/content", {}, fetcher); + const result = await extract( + "https://somewhere.com/path/to/no/content", + {}, + fetcher, + ); assertEquals(result, null); }); @@ -72,22 +85,21 @@ Deno.test("extract - html string directly", async () => { assertEquals((result as Record).description, expDesc); }); -Deno.test("extract with modified sanitize-html options", async () => { - const currentSanitizeOptions = getSanitizeHtmlOptions(); - - setSanitizeHtmlOptions({ - ...currentSanitizeOptions, +Deno.test("extract with custom allowed attributes via ParserOptions", async () => { + const html = Deno.readTextFileSync( + "tests/test-data/article-with-classes-attributes.html", + ); + const result = await extract(html, { + allowedTags: defaultAllowedTags, allowedAttributes: { - ...currentSanitizeOptions.allowedAttributes as Record, + ...defaultAllowedAttributes, code: ["class"], div: ["class"], }, - allowedClasses: { - code: ["language-*", "lang-*"], - }, + allowedIframeDomains: defaultAllowedIframeDomains, }); - - const html = Deno.readTextFileSync("tests/test-data/article-with-classes-attributes.html"); - const result = await extract(html); - assertEquals((result as Record).content.includes('code class="lang-js"'), true); + assertEquals( + (result as Record).content.includes('code class="lang-js"'), + true, + ); }); diff --git a/tests/parseFromHtml.test.ts b/tests/parseFromHtml.test.ts index da204df..b61b25c 100644 --- a/tests/parseFromHtml.test.ts +++ b/tests/parseFromHtml.test.ts @@ -2,7 +2,10 @@ import { assertEquals } from "@std/assert"; import { extractFromHtml as parseFromHtml } from "../src/main.ts"; -import { addTransformations, removeTransformations } from "../src/utils/transformation.ts"; +import { + addTransformations, + removeTransformations, +} from "../src/utils/transformation.ts"; const expDesc = [ "Navigation here Few can name a rational peach that isn't a conscientious goldfish!", @@ -29,31 +32,60 @@ Deno.test("parseFromHtml - webpage with no main article", async () => { }); Deno.test("parseFromHtml - webpage with very short article", async () => { - const html = Deno.readTextFileSync("tests/test-data/html-too-short-article.html"); + const html = Deno.readTextFileSync( + "tests/test-data/html-too-short-article.html", + ); const result = await parseFromHtml(html, "abcd"); assertEquals(result, null); }); Deno.test("parseFromHtml - webpage with article but no source", async () => { - const html = Deno.readTextFileSync("tests/test-data/html-article-no-source.html"); + const html = Deno.readTextFileSync( + "tests/test-data/html-article-no-source.html", + ); const result = await parseFromHtml(html); assertEquals((result as Record).source, "somewhere.any"); }); Deno.test("parseFromHtml - webpage with data-src in img tag", async () => { - const html = Deno.readTextFileSync("tests/test-data/html-article-with-data-src.html"); + const html = Deno.readTextFileSync( + "tests/test-data/html-article-with-data-src.html", + ); const result = await parseFromHtml(html); - assertEquals((result as Record).content.includes(''), true); - assertEquals((result as Record).content.includes(''), true); + assertEquals( + (result as Record).content.includes( + '', + ), + true, + ); + assertEquals( + (result as Record).content.includes( + '', + ), + true, + ); }); Deno.test("parseFromHtml - regular article", async () => { const html = Deno.readTextFileSync("tests/test-data/regular-article.html"); - const result = await parseFromHtml(html, "https://somewhere.com/path/to/article"); + const result = await parseFromHtml( + html, + "https://somewhere.com/path/to/article", + ); assertEquals((result as Record).title, "Article title here"); assertEquals((result as Record).description, expDesc); - assertEquals((result as Record).content.includes(''), true); - assertEquals((result as Record).content.includes(''), true); + assertEquals( + (result as Record).content.includes( + '', + ), + true, + ); + assertEquals( + (result as Record).content.includes( + '', + ), + true, + ); }); Deno.test("parseFromHtml - multi transforms", async () => { @@ -66,7 +98,10 @@ Deno.test("parseFromHtml - multi transforms", async () => { document.querySelectorAll("a").forEach((node: Element) => { const sHtml = node.innerHTML; const link = node.getAttribute("href"); - node.parentNode!.replaceChild(document.createTextNode(`[link url="${link}"]${sHtml}[/link]`), node); + node.parentNode!.replaceChild( + document.createTextNode(`[link url="${link}"]${sHtml}[/link]`), + node, + ); }); return document; }, @@ -89,8 +124,23 @@ Deno.test("parseFromHtml - multi transforms", async () => { const url = "https://need-transform.tld/path/to/article"; const result = await parseFromHtml(html, url); assertEquals((result as Record).title, "Article title here"); - assertEquals((result as Record).content.includes(''), false); - assertEquals((result as Record).content.includes('[link url="https://vnn.vn/dict/watermelon"]watermelon[/link]'), true); - assertEquals((result as Record).content.includes("in its own way"), true); + assertEquals( + (result as Record).content.includes( + '', + ), + false, + ); + assertEquals( + (result as Record).content.includes( + '[link url="https://vnn.vn/dict/watermelon"]watermelon[/link]', + ), + true, + ); + assertEquals( + (result as Record).content.includes( + "in its own way", + ), + true, + ); removeTransformations(); }); diff --git a/tests/retrieve.test.ts b/tests/retrieve.test.ts index 7a5ddae..056c545 100644 --- a/tests/retrieve.test.ts +++ b/tests/retrieve.test.ts @@ -24,7 +24,10 @@ Deno.test("retrieve - good source", async () => { Deno.test("retrieve - good source with \\r\\n", async () => { const url = "https://some.where/good/page"; - const fetcher = createMockFetcher("\n\r\r\n\n
this is content
\n\r\r\n\n", "text/html"); + const fetcher = createMockFetcher( + "\n\r\r\n\n
this is content
\n\r\r\n\n", + "text/html", + ); const buffer = await retrieve(url, fetcher); const html = new TextDecoder().decode(buffer).trim(); assertEquals(html, "
this is content
"); @@ -32,7 +35,10 @@ Deno.test("retrieve - good source with \\r\\n", async () => { Deno.test("retrieve - custom fetcher", async () => { const url = "https://some.where/good/source-with-fetcher"; - const innerFetcher = createMockFetcher("
this is content
", "text/html"); + const innerFetcher = createMockFetcher( + "
this is content
", + "text/html", + ); const myFetcher = (fetchUrl: string) => { assertEquals(fetchUrl, url); return innerFetcher(); diff --git a/tests/test-data/article-with-classes-attributes.html b/tests/test-data/article-with-classes-attributes.html index 155dc66..b415bcf 100644 --- a/tests/test-data/article-with-classes-attributes.html +++ b/tests/test-data/article-with-classes-attributes.html @@ -8,24 +8,33 @@ - + - + - - + + - - + + - - + + - + @@ -38,7 +47,8 @@

Article title here

-
Few can name a rational peach that isn't a conscientious goldfish! One cannot separate snakes from plucky pomegranates? Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs. Some posit the enchanting tiger to be less than confident. The literature would have us believe that an impartial turtle is not but a hippopotamus. Unfortunately, that is wrong; on the contrary, those cows are nothing more than pandas! The chicken is a shark; A turtle can hardly be considered a kind horse without also being a pomegranate. Zebras are witty persimmons.
+
Few can name a rational peach that isn't a conscientious goldfish! One cannot separate snakes from plucky pomegranates? Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs. Some posit the enchanting tiger to be less than confident. The literature would have us believe that an impartial turtle is not but a hippopotamus. Unfortunately, that is wrong; on the contrary, those cows are nothing more than pandas! The chicken is a shark; A turtle can hardly be considered a kind horse without also being a pomegranate. Zebras are witty persimmons.

Those cheetahs are nothing more than dogs. A watermelon is an exuberant kangaroo. An octopus is the tangerine of a grapes? The cherry is a shark. Recent controversy aside, they were lost without the cheerful plum that composed their fox. As far as we can estimate, one cannot separate camels from dynamic hamsters. Those tigers are nothing more than cows! A cow is a squirrel from the right perspective. Their banana was, in this moment, a helpful bear.

The first fair dog is, in its own way, a lemon.

diff --git a/tests/test-data/html-article-no-source.html b/tests/test-data/html-article-no-source.html index 1223708..daee82c 100644 --- a/tests/test-data/html-article-no-source.html +++ b/tests/test-data/html-article-no-source.html @@ -1,13 +1,13 @@ - - - - - TechNews - - -
+ + + + + TechNews + + +
Few can name an easygoing deer that isn't a frank hamster. One cannot separate tangerines from excellent oranges. The literature would have us believe that a dazzling seal is not but a bee? Some posit the nice strawberry to be less than fair-minded. Washing and polishing the car,a monkey is the monkey of a turtle; What we don't know for sure is whether or not the spiders could be said to resemble ambitious cherries. Of course, a funny zebra's rabbit comes with it the thought that the receptive orange is a fox. If this was somewhat unclear, a sheep is a strawberry's chimpanzee. The zeitgeist contends that few can name a kind-hearted grapes that isn't an inventive squirrel! Few can name a friendly deer that isn't a decorous grapes? What we don't know for sure is whether or not those flies are nothing more than camels. @@ -18,5 +18,5 @@ Shouting with happiness, a courageous elephant is a duck of the mind? Some posit the upbeat hippopotamus to be less than enchanting. It's an undeniable fact, really; authors often misinterpret the grape as an endurable rabbit, when in actuality it feels more like a tough dolphin. We know that a cherry can hardly be considered a responsible apricot without also being a nectarine.
- + diff --git a/tests/test-data/html-article-with-data-src.html b/tests/test-data/html-article-with-data-src.html index 9a51ac0..b6359ff 100644 --- a/tests/test-data/html-article-with-data-src.html +++ b/tests/test-data/html-article-with-data-src.html @@ -1,13 +1,13 @@ - + - + Article title TechNews - - -
+ + +
Few can name an easygoing deer that isn't a frank hamster. One cannot separate tangerines from excellent oranges. @@ -15,6 +15,6 @@ The literature would have us believe that a dazzling seal is not but a bee? Some posit the nice strawberry to be less than fair-minded. Washing and polishing the car,a monkey is the monkey of a turtle; -
- +
+ diff --git a/tests/test-data/html-no-article.html b/tests/test-data/html-no-article.html index 1212ecc..0da49e1 100644 --- a/tests/test-data/html-no-article.html +++ b/tests/test-data/html-no-article.html @@ -1,8 +1,8 @@ - - - + + + Article title here - ArticleParser @@ -10,9 +10,9 @@ - + - - - + + diff --git a/tests/test-data/html-no-link.html b/tests/test-data/html-no-link.html index 0d5f46d..e7a59a5 100644 --- a/tests/test-data/html-no-link.html +++ b/tests/test-data/html-no-link.html @@ -7,7 +7,8 @@ - + @@ -19,7 +20,8 @@

Article title here

-
Few can name a rational peach that isn't a conscientious goldfish! One cannot separate snakes from plucky pomegranates? Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs. Some posit the enchanting tiger to be less than confident. The literature would have us believe that an impartial turtle is not but a hippopotamus. Unfortunately, that is wrong; on the contrary, those cows are nothing more than pandas! The chicken is a shark; A turtle can hardly be considered a kind horse without also being a pomegranate. Zebras are witty persimmons.
+
Few can name a rational peach that isn't a conscientious goldfish! One cannot separate snakes from plucky pomegranates? Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs. Some posit the enchanting tiger to be less than confident. The literature would have us believe that an impartial turtle is not but a hippopotamus. Unfortunately, that is wrong; on the contrary, those cows are nothing more than pandas! The chicken is a shark; A turtle can hardly be considered a kind horse without also being a pomegranate. Zebras are witty persimmons.

Those cheetahs are nothing more than dogs. A watermelon is an exuberant kangaroo. An octopus is the tangerine of a grapes? The cherry is a shark. Recent controversy aside, they were lost without the cheerful plum that composed their fox. As far as we can estimate, one cannot separate camels from dynamic hamsters. Those tigers are nothing more than cows! A cow is a squirrel from the right perspective. Their banana was, in this moment, a helpful bear.

The first fair dog is, in its own way, a lemon.

diff --git a/tests/test-data/html-no-title.html b/tests/test-data/html-no-title.html index f2f7932..c8e8efd 100644 --- a/tests/test-data/html-no-title.html +++ b/tests/test-data/html-no-title.html @@ -1,9 +1,9 @@ - - - - + + + +
Page header here
@@ -13,7 +13,8 @@

Article title here

-
Few can name a rational peach that isn't a conscientious goldfish! One cannot separate snakes from plucky pomegranates? Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs. Some posit the enchanting tiger to be less than confident. The literature would have us believe that an impartial turtle is not but a hippopotamus. Unfortunately, that is wrong; on the contrary, those cows are nothing more than pandas! The chicken is a shark; A turtle can hardly be considered a kind horse without also being a pomegranate. Zebras are witty persimmons.
+
Few can name a rational peach that isn't a conscientious goldfish! One cannot separate snakes from plucky pomegranates? Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs. Some posit the enchanting tiger to be less than confident. The literature would have us believe that an impartial turtle is not but a hippopotamus. Unfortunately, that is wrong; on the contrary, those cows are nothing more than pandas! The chicken is a shark; A turtle can hardly be considered a kind horse without also being a pomegranate. Zebras are witty persimmons.

Those cheetahs are nothing more than dogs. A watermelon is an exuberant kangaroo. An octopus is the tangerine of a grapes? The cherry is a shark. Recent controversy aside, they were lost without the cheerful plum that composed their fox. As far as we can estimate, one cannot separate camels from dynamic hamsters. Those tigers are nothing more than cows! A cow is a squirrel from the right perspective. Their banana was, in this moment, a helpful bear.

The first fair dog is, in its own way, a lemon.

diff --git a/tests/test-data/html-too-short-article.html b/tests/test-data/html-too-short-article.html index 4982be3..9a7632e 100644 --- a/tests/test-data/html-too-short-article.html +++ b/tests/test-data/html-too-short-article.html @@ -1,15 +1,15 @@ - - - - - TechNews - - -
+ + + + + TechNews + + +
Hello world, this is too short, right?

The first fair dog is, in its own way, a lemon.

- + diff --git a/tests/test-data/regular-article-date-itemprop.html b/tests/test-data/regular-article-date-itemprop.html index c23a4e9..fcd5f55 100644 --- a/tests/test-data/regular-article-date-itemprop.html +++ b/tests/test-data/regular-article-date-itemprop.html @@ -8,23 +8,30 @@ - + - + - - + + - - + + - + @@ -37,9 +44,11 @@

Article title here

- + -
Few can name a rational peach that isn't a conscientious goldfish! One cannot separate snakes from plucky pomegranates? Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs. Some posit the enchanting tiger to be less than confident. The literature would have us believe that an impartial turtle is not but a hippopotamus. Unfortunately, that is wrong; on the contrary, those cows are nothing more than pandas! The chicken is a shark; A turtle can hardly be considered a kind horse without also being a pomegranate. Zebras are witty persimmons.
+
Few can name a rational peach that isn't a conscientious goldfish! One cannot separate snakes from plucky pomegranates? Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs. Some posit the enchanting tiger to be less than confident. The literature would have us believe that an impartial turtle is not but a hippopotamus. Unfortunately, that is wrong; on the contrary, those cows are nothing more than pandas! The chicken is a shark; A turtle can hardly be considered a kind horse without also being a pomegranate. Zebras are witty persimmons.

Those cheetahs are nothing more than dogs. A watermelon is an exuberant kangaroo. An octopus is the tangerine of a grapes? The cherry is a shark. Recent controversy aside, they were lost without the cheerful plum that composed their fox. As far as we can estimate, one cannot separate camels from dynamic hamsters. Those tigers are nothing more than cows! A cow is a squirrel from the right perspective. Their banana was, in this moment, a helpful bear.

The first fair dog is, in its own way, a lemon.

diff --git a/tests/test-data/regular-article-date-span.html b/tests/test-data/regular-article-date-span.html index e6c13fc..89e08b7 100644 --- a/tests/test-data/regular-article-date-span.html +++ b/tests/test-data/regular-article-date-span.html @@ -8,23 +8,30 @@ - + - + - - + + - - + + - + @@ -37,9 +44,10 @@

Article title here

- Published at 11/09/2024 07h33 am + Published at 11/09/2024 07h33 am -
Few can name a rational peach that isn't a conscientious goldfish! One cannot separate snakes from plucky pomegranates? Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs. Some posit the enchanting tiger to be less than confident. The literature would have us believe that an impartial turtle is not but a hippopotamus. Unfortunately, that is wrong; on the contrary, those cows are nothing more than pandas! The chicken is a shark; A turtle can hardly be considered a kind horse without also being a pomegranate. Zebras are witty persimmons.
+
Few can name a rational peach that isn't a conscientious goldfish! One cannot separate snakes from plucky pomegranates? Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs. Some posit the enchanting tiger to be less than confident. The literature would have us believe that an impartial turtle is not but a hippopotamus. Unfortunately, that is wrong; on the contrary, those cows are nothing more than pandas! The chicken is a shark; A turtle can hardly be considered a kind horse without also being a pomegranate. Zebras are witty persimmons.

Those cheetahs are nothing more than dogs. A watermelon is an exuberant kangaroo. An octopus is the tangerine of a grapes? The cherry is a shark. Recent controversy aside, they were lost without the cheerful plum that composed their fox. As far as we can estimate, one cannot separate camels from dynamic hamsters. Those tigers are nothing more than cows! A cow is a squirrel from the right perspective. Their banana was, in this moment, a helpful bear.

The first fair dog is, in its own way, a lemon.

diff --git a/tests/test-data/regular-article-date-time.html b/tests/test-data/regular-article-date-time.html index d1e1638..6b32cd2 100644 --- a/tests/test-data/regular-article-date-time.html +++ b/tests/test-data/regular-article-date-time.html @@ -8,23 +8,30 @@ - + - + - - + + - - + + - + @@ -37,9 +44,11 @@

Article title here

- + -
Few can name a rational peach that isn't a conscientious goldfish! One cannot separate snakes from plucky pomegranates? Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs. Some posit the enchanting tiger to be less than confident. The literature would have us believe that an impartial turtle is not but a hippopotamus. Unfortunately, that is wrong; on the contrary, those cows are nothing more than pandas! The chicken is a shark; A turtle can hardly be considered a kind horse without also being a pomegranate. Zebras are witty persimmons.
+
Few can name a rational peach that isn't a conscientious goldfish! One cannot separate snakes from plucky pomegranates? Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs. Some posit the enchanting tiger to be less than confident. The literature would have us believe that an impartial turtle is not but a hippopotamus. Unfortunately, that is wrong; on the contrary, those cows are nothing more than pandas! The chicken is a shark; A turtle can hardly be considered a kind horse without also being a pomegranate. Zebras are witty persimmons.

Those cheetahs are nothing more than dogs. A watermelon is an exuberant kangaroo. An octopus is the tangerine of a grapes? The cherry is a shark. Recent controversy aside, they were lost without the cheerful plum that composed their fox. As far as we can estimate, one cannot separate camels from dynamic hamsters. Those tigers are nothing more than cows! A cow is a squirrel from the right perspective. Their banana was, in this moment, a helpful bear.

The first fair dog is, in its own way, a lemon.

diff --git a/tests/test-data/regular-article-json-ld.html b/tests/test-data/regular-article-json-ld.html index c48ad9d..a4960d7 100644 --- a/tests/test-data/regular-article-json-ld.html +++ b/tests/test-data/regular-article-json-ld.html @@ -6,36 +6,43 @@ Article title here - ArticleParser - + - + - + } + ] + - + - - + + - + @@ -70,7 +80,8 @@

Article title here

-
Few can name a rational peach that isn't a conscientious goldfish! One cannot separate snakes from plucky pomegranates? Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs. Some posit the enchanting tiger to be less than confident. The literature would have us believe that an impartial turtle is not but a hippopotamus. Unfortunately, that is wrong; on the contrary, those cows are nothing more than pandas! The chicken is a shark; A turtle can hardly be considered a kind horse without also being a pomegranate. Zebras are witty persimmons.
+
Few can name a rational peach that isn't a conscientious goldfish! One cannot separate snakes from plucky pomegranates? Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs. Some posit the enchanting tiger to be less than confident. The literature would have us believe that an impartial turtle is not but a hippopotamus. Unfortunately, that is wrong; on the contrary, those cows are nothing more than pandas! The chicken is a shark; A turtle can hardly be considered a kind horse without also being a pomegranate. Zebras are witty persimmons.

Those cheetahs are nothing more than dogs. A watermelon is an exuberant kangaroo. An octopus is the tangerine of a grapes? The cherry is a shark. Recent controversy aside, they were lost without the cheerful plum that composed their fox. As far as we can estimate, one cannot separate camels from dynamic hamsters. Those tigers are nothing more than cows! A cow is a squirrel from the right perspective. Their banana was, in this moment, a helpful bear.

The first fair dog is, in its own way, a lemon.

diff --git a/tests/test-data/regular-article.html b/tests/test-data/regular-article.html index ab68045..0749b7e 100644 --- a/tests/test-data/regular-article.html +++ b/tests/test-data/regular-article.html @@ -8,25 +8,34 @@ - + - + - - + + - - + + - - + + - + @@ -39,7 +48,8 @@

Article title here

-
Few can name a rational peach that isn't a conscientious goldfish! One cannot separate snakes from plucky pomegranates? Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs. Some posit the enchanting tiger to be less than confident. The literature would have us believe that an impartial turtle is not but a hippopotamus. Unfortunately, that is wrong; on the contrary, those cows are nothing more than pandas! The chicken is a shark; A turtle can hardly be considered a kind horse without also being a pomegranate. Zebras are witty persimmons.
+
Few can name a rational peach that isn't a conscientious goldfish! One cannot separate snakes from plucky pomegranates? Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs. Some posit the enchanting tiger to be less than confident. The literature would have us believe that an impartial turtle is not but a hippopotamus. Unfortunately, that is wrong; on the contrary, those cows are nothing more than pandas! The chicken is a shark; A turtle can hardly be considered a kind horse without also being a pomegranate. Zebras are witty persimmons.

Those cheetahs are nothing more than dogs. A watermelon is an exuberant kangaroo. An octopus is the tangerine of a grapes? The cherry is a shark. Recent controversy aside, they were lost without the cheerful plum that composed their fox. As far as we can estimate, one cannot separate camels from dynamic hamsters. Those tigers are nothing more than cows! A cow is a squirrel from the right perspective. Their banana was, in this moment, a helpful bear.

The first fair dog is, in its own way, a lemon.

diff --git a/tests/test-data/vnn-article.html b/tests/test-data/vnn-article.html index 41ab935..65a875a 100644 --- a/tests/test-data/vnn-article.html +++ b/tests/test-data/vnn-article.html @@ -8,24 +8,31 @@ - + - + - - + + - - + + - + @@ -38,7 +45,8 @@

Article title here

-
Few can name a rational peach that isn't a conscientious goldfish! One cannot separate snakes from plucky pomegranates? Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs. Some posit the enchanting tiger to be less than confident. The literature would have us believe that an impartial turtle is not but a hippopotamus. Unfortunately, that is wrong; on the contrary, those cows are nothing more than pandas! The chicken is a shark; A turtle can hardly be considered a kind horse without also being a pomegranate. Zebras are witty persimmons.
+
Few can name a rational peach that isn't a conscientious goldfish! One cannot separate snakes from plucky pomegranates? Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs. Some posit the enchanting tiger to be less than confident. The literature would have us believe that an impartial turtle is not but a hippopotamus. Unfortunately, that is wrong; on the contrary, those cows are nothing more than pandas! The chicken is a shark; A turtle can hardly be considered a kind horse without also being a pomegranate. Zebras are witty persimmons.

Those cheetahs are nothing more than dogs. A watermelon is an exuberant kangaroo. An octopus is the tangerine of a grapes? The cherry is a shark. Recent controversy aside, they were lost without the cheerful plum that composed their fox. As far as we can estimate, one cannot separate camels from dynamic hamsters. Those tigers are nothing more than cows! A cow is a squirrel from the right perspective. Their banana was, in this moment, a helpful bear.

diff --git a/tests/transformation.test.ts b/tests/transformation.test.ts index 65dc18e..2e9202d 100644 --- a/tests/transformation.test.ts +++ b/tests/transformation.test.ts @@ -3,11 +3,11 @@ import { assertEquals } from "@std/assert"; import { addTransformations, - removeTransformations, - getTransformations, - findTransformations, - execPreParser, execPostParser, + execPreParser, + findTransformations, + getTransformations, + removeTransformations, } from "../src/utils/transformation.ts"; Deno.test("add one transformation object", () => { @@ -71,7 +71,10 @@ Deno.test("add transformation object without valid patterns", () => { Deno.test("get all transformations", () => { const result = getTransformations(); assertEquals(result.length, 3); - assertEquals(String(result[0].patterns[0]), String(/http(s?):\/\/([\w]+.)?def.tld\/*/)); + assertEquals( + String(result[0].patterns[0]), + String(/http(s?):\/\/([\w]+.)?def.tld\/*/), + ); }); Deno.test("remove one transformation", () => { @@ -104,7 +107,10 @@ Deno.test("remove one transformation", () => { Deno.test("get all transformations again", () => { const result = getTransformations(); assertEquals(result.length, 4); - assertEquals(String(result[3].patterns[1]), String(/http(s?):\/\/rst.inc\/*/)); + assertEquals( + String(result[3].patterns[1]), + String(/http(s?):\/\/rst.inc\/*/), + ); }); Deno.test("find transformations", () => { @@ -153,8 +159,14 @@ Deno.test("execPreParser", () => {
`; const result = execPreParser(html, "https://xyz.com/article"); - assertEquals(result.includes("hi user, this is an advertisement element"), true); - assertEquals(result.includes('
free product now!
'), false); + assertEquals( + result.includes("hi user, this is an advertisement element"), + true, + ); + assertEquals( + result.includes('
free product now!
'), + false, + ); }); Deno.test("execPostParser", () => { From 7dd0cd8affd22c33d124dde8f2d306523f800f45 Mon Sep 17 00:00:00 2001 From: Dong Nguyen Date: Thu, 6 Aug 2026 18:36:38 +0700 Subject: [PATCH 5/6] Rename jsr release --- .github/workflows/{release.yml => jsr-release.yml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/{release.yml => jsr-release.yml} (96%) diff --git a/.github/workflows/release.yml b/.github/workflows/jsr-release.yml similarity index 96% rename from .github/workflows/release.yml rename to .github/workflows/jsr-release.yml index 028f8df..f5dfec6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/jsr-release.yml @@ -1,4 +1,4 @@ -name: Release +name: jsr-release on: release: From 6cd7284dbabad731bf8e6f8c4003782861a8d156 Mon Sep 17 00:00:00 2001 From: Dong Nguyen Date: Thu, 6 Aug 2026 19:02:53 +0700 Subject: [PATCH 6/6] v9.0.0 --- deno.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deno.json b/deno.json index fae3b00..cb0a6ab 100644 --- a/deno.json +++ b/deno.json @@ -1,6 +1,6 @@ { "name": "@extractus/article-extractor", - "version": "9.0.0-rc1", + "version": "9.0.0", "description": "To extract main article from given URL", "homepage": "https://github.com/extractus/article-extractor", "repository": {