diff --git a/.aiignore b/.aiignore deleted file mode 100644 index c880c25b..00000000 --- a/.aiignore +++ /dev/null @@ -1,15 +0,0 @@ -node_modules -coverage -coverage.lcov - -package-lock.json -pnpm-lock.yaml -bun.lock - -.env - -dist -storage - -# AI Session Files (Private Context) -.sessions diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index d6c6a5fa..ac7a6c46 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -1,38 +1,27 @@ -# GitHub actions -# https://docs.github.com/en/free-pro-team@latest/actions - name: ci-test on: [push, pull_request] jobs: test: - runs-on: ubuntu-latest - strategy: matrix: - node_version: [22.x, 24.x, 25.x] + deno-version: ["lts", "canary"] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 + + - name: Setup Deno + uses: denoland/setup-deno@v2 + with: + deno-version: ${{ matrix.deno-version }} - - name: setup Node.js v${{ matrix.node_version }} - uses: actions/setup-node@v6 - with: - node-version: ${{ matrix.node_version }} + - name: lint + run: deno lint - - name: run npm scripts - run: | - npm install - npm run lint - #npm run build --if-present - npm run test + - name: test + run: deno test --allow-all - - name: cache node modules - uses: actions/cache@v5 - with: - path: ~/.npm - key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} - restore-keys: | - ${{ runner.os }}-node- + - name: audit + run: deno audit diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml deleted file mode 100644 index 5547051d..00000000 --- a/.github/workflows/codeql-analysis.yml +++ /dev/null @@ -1,70 +0,0 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# -name: "CodeQL" - -on: - push: - branches: [ main ] - pull_request: - # The branches below must be a subset of the branches above - branches: [ main ] - schedule: - - cron: '33 18 * * 3' - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - permissions: - actions: read - contents: read - security-events: write - - strategy: - fail-fast: false - matrix: - language: [ 'javascript' ] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] - # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support - - steps: - - name: Checkout repository - uses: actions/checkout@v6 - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main - - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v2 - - # โ„น๏ธ Command-line programs to run using the OS shell. - # ๐Ÿ“š See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun - - # โœ๏ธ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language - - #- run: | - # make bootstrap - # make release - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/jsr-release.yml b/.github/workflows/jsr-release.yml new file mode 100644 index 00000000..f5dfec62 --- /dev/null +++ b/.github/workflows/jsr-release.yml @@ -0,0 +1,32 @@ +name: jsr-release + +on: + release: + types: [published] + +jobs: + publish: + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + + steps: + - uses: actions/checkout@v7 + + - name: Setup Deno + uses: denoland/setup-deno@v2 + with: + deno-version: lts + + - name: lint + run: deno lint + + - name: test + run: deno test --allow-all + + - name: audit + run: deno audit + + - name: publish to jsr + run: deno publish diff --git a/.gitignore b/.gitignore index 971acbc7..3a7c626f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,24 +1,5 @@ -# Logs -logs -*.log -*.debug - -# Runtime data -*.pid -*.seed - node_modules -coverage -.nyc_output - -.DS_Store -yarn.lock -coverage.lcov -pnpm-lock.yaml -lcov.info -bun.lock -deno.lock -evaluation +.env -.sessions +npm/ diff --git a/.npmignore b/.npmignore deleted file mode 100644 index a0f6f850..00000000 --- a/.npmignore +++ /dev/null @@ -1,10 +0,0 @@ -node_modules -coverage -.github -pnpm-lock.yaml -examples -test-data -lcov.info - -.aiignore -.sessions diff --git a/AGENTS.md b/AGENTS.md deleted file mode 100644 index bfc44e1a..00000000 --- a/AGENTS.md +++ /dev/null @@ -1,113 +0,0 @@ -# AI Agent Instructions - -Coding guidelines for AI agents working in this project. - -## Philosophy - -- Minimalism. Simple is better. KISS (Keep It Simple, Stupid). -- Clean code, easy to read, easy to delete. -- Functional Programming โ€” pure functions, immutability, no side effects. -- MVP mindset โ€” deliver the smallest thing that works, then iterate. - -## Security Rules (CRITICAL โ€” no exceptions) - -- NEVER output or request .env and example.env file contents -- NEVER hardcode API credentials, secret tokens, private keys or passwords in source code -- NEVER send sensitive data to external AI services -- Follow `.aiignore` and `.gitignore` for excluded files โ€” do not read or reference them -- When asking for help, sanitize data (replace real IDs, emails, tokens with placeholders) -- Do not log sensitive information - -## Coding Standards (Strict) -- Language: JavaScript (ESM syntax). No TypeScript. -- Style: No semicolons, single quotes, 2-space indentation. -- Respect `eslint.config.js` โ€” do not suggest rule changes -- Patterns: - - Functional Programming only. No Classes or OOP. - - Arrow functions are preferred. - - Maximum 3 parameters per function. Use objects for more. -- Naming: camelCase for variables/functions, SNAKE_CASE for constants. -- Documentation: - - Add JSDocs before all functions and exported variables. - - Language: Use American English for all comments and JSDocs. - - Constraint: NEVER use Vietnamese or other languages in the source code. - -### Error Handling - -- Handle errors explicitly โ€” never swallow silently -- Use try/catch with proper logging -- Return null or throw meaningful errors - -```javascript -export const send = async (params) => { - try { - const response = await ai.ask(params) - logger.info(`send() -> success: ${response.id}`) - return response - } catch (err) { - logger.error(`send() -> failed: ${err.message}`) - console.error(err) - return null - } -} -``` - -## Testing Standards - -- Write tests for critical business logic, all error cases -- Use simple test runners (node:test, bun:test, vitest) -- No complex mocking frameworks unless necessary -- Tests live alongside source: `[module].test.js` next to `[module].js` - -## Dependency Rules - -- Prefer built-in APIs over external packages -- Before adding dependency, explain: - - Why it is needed - - Alternatives considered - - Bundle size impact -- Never add dependency for trivial utilities -- Avoid packages with large dependency trees - -## Architecture Rules - -- Do NOT change existing project architecture without explicit approval -- Do NOT move or rename core modules unless requested -- Respect module boundaries -- Avoid cross-module coupling -- New modules must follow existing folder structure - -## When Making Changes - -1. Read existing patterns first -2. Follow current coding style strictly -3. Keep dependencies minimal -4. Handle errors explicitly -5. Add JSDoc comments for new functions -6. Run `npm run lint` before committing -7. Do NOT refactor unrelated code -8. Do NOT modify working code outside task scope -9. Prefer minimal diff changes -10. Preserve existing behavior unless explicitly requested - -## When in Doubt - -- Ask for clarification before generating code -- State your assumption explicitly if proceeding without confirmation -- Prefer doing less and asking over doing more and guessing - -## Git Workflow - -- Work only inside the current branch -- Do NOT create or delete branches -- Do NOT rewrite git history -- Do NOT modify commit messages -- Changes must correspond to the current issue - -## Agent References - -Reference these URLs when working on related topics: - -- Bun: https://bun.sh/llms-full.txt - ---- diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8cfca770..c28b4dbf 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,8 +10,7 @@ As an OSS, it's better to follow the Unix philosophy: "do one thing and do it we Please avoid using libaries other than those available in the standard library, unless necessary. -This library needs to be simple and flexible to run on multiple platforms such as Deno, Bun, or even browser. - +This library needs to be simple and flexible to run on multiple platforms (Deno, Node, Bun). ## Coding convention @@ -22,34 +21,21 @@ Make sure your code lints before opening a pull request. cd article-extractor # check coding convention issue -npm run lint +deno lint # auto fix coding convention issue -npm run lint:fix +deno lint --fix ``` -*When you run `npm test`, the linting process will be triggered at first.* - - ## Testing Be sure to run the unit test suite before opening a pull request. An example test run is shown below. ```bash cd article-extractor -npm test +deno test --allow-all ``` -![article-extractor unit test](https://i.imgur.com/TbRCUSS.png?110222) - -If test coverage decreased, please check test scripts and try to improve this number. - - -## Documentation - -If you've changed APIs, please update README and [the examples](examples). - - ## Clean commit histories When you open a pull request, please ensure the commit history is clean. @@ -58,12 +44,6 @@ Squash the commits into logical blocks, perhaps a single commit if that makes se What you want to avoid is commits such as "WIP" and "fix test" in the history. This is so we keep history on master clean and straightforward. -For people new to git, please refer the following guides: - -- [Writing good commit messages](https://github.com/erlang/otp/wiki/writing-good-commit-messages) -- [Commit Message Guidelines](https://gist.github.com/robertpainsi/b632364184e70900af4ab688decf6f53) - - ## License By contributing to `@extractus/article-extractor`, you agree that your contributions will be licensed under its [MIT license](LICENSE). diff --git a/README.md b/README.md index 70020954..3db34aec 100644 --- a/README.md +++ b/README.md @@ -2,495 +2,365 @@ Extract main article, main image and meta data from URL. +[![JSR](https://jsr.io/badges/@extractus/article-extractor)](https://jsr.io/@extractus/article-extractor) [![npm version](https://badge.fury.io/js/@extractus%2Farticle-extractor.svg)](https://badge.fury.io/js/@extractus%2Farticle-extractor) -![CodeQL](https://github.com/extractus/article-extractor/workflows/CodeQL/badge.svg) ![CI test](https://github.com/extractus/article-extractor/workflows/ci-test/badge.svg) -(This library is derived from [article-parser](https://www.npmjs.com/package/article-parser) renamed.) +## Install -## Demo +### Deno -- [Give it a try!](https://extractus.pwshub.com/article) +```bash +deno add jsr:@extractus/article-extractor +``` -## Install +### Node.js / Bun ```bash -# bun -bun add @extractus/article-extractor - -# npm -npm i @extractus/article-extractor +pnpm add jsr:@extractus/article-extractor +# or +npx jsr add @extractus/article-extractor +# or +bunx jsr add @extractus/article-extractor +``` -# pnpm -pnpm install @extractus/article-extractor +Alternatively, install from npm: -# yarn -yarn add @extractus/article-extractor +```bash +npm install @extractus/article-extractor +# or +bun add @extractus/article-extractor ``` ## Usage ```ts -import { extract } from '@extractus/article-extractor' +import { extract } from "jsr:@extractus/article-extractor"; -const data = await extract(ARTICLE_URL) -console.log(data) +const data = await extract("https://example.com/article"); +console.log(data); ``` ## APIs -- [extract()](#extract) -- [extractFromHtml()](#extractfromhtml) +- [`extract()`](#extract) +- [`extractFromHtml()`](#extractfromhtml) - [Transformations](#transformations) - - [`transformation` object](#transformation-object) - - [.addTransformations](#addtransformationsobject-transformation--array-transformations) - - [.removeTransformations](#removetransformationsarray-patterns) + - [`Transformation` object](#transformation-object) + - [`addTransformations()`](#addtransformationstransformation--transformation) + - [`removeTransformations()`](#removetransformationspatterns-regexp) - [Priority order](#priority-order) -- [`sanitize-html`'s options](#sanitize-htmls-options) +- [Content sanitization options](#content-sanitization-options) --- ### `extract()` -Load and extract article data. Return a Promise object. +Load and extract article data from a URL or HTML string. #### Syntax ```ts -extract(String input) -extract(String input, Object parserOptions) -extract(String input, Object parserOptions, Object fetchOptions) +extract(input: string): Promise +extract(input: string, parserOptions?: ParserOptions): Promise +extract(input: string, parserOptions?: ParserOptions, fetcher?: Fetcher): Promise ``` Example: -```js -import { extract } from '@extractus/article-extractor' - -const input = 'https://www.cnbc.com/2022/09/21/what-another-major-rate-hike-by-the-federal-reserve-means-to-you.html' +```ts +import { extract } from "jsr:@extractus/article-extractor"; -// here we use top-level await, assume current platform supports it try { - const article = await extract(input) - console.log(article) + const article = await extract("https://example.com/some-article"); + console.log(article); } catch (err) { - console.error(err) + console.error(err); } ``` -The result - `article` - can be `null` or an object with the following structure: +The result can be `null` (when no article found) or an `ArticleData` object: ```ts -{ - url: String, - title: String, - description: String, - image: String, - author: String, - favicon: String, - content: String, - published: Date String, - type: String, // page type - source: String, // original publisher - links: Array, // list of alternative links - ttr: Number, // time to read in second, 0 = unknown +interface ArticleData { + url?: string; // best resolved URL + links?: string[]; // alternative URLs (canonical, shortlink, amphtml) + title?: string; // article title + description?: string; // short description / excerpt + image?: string; // main image URL + favicon?: string; // site favicon URL + author?: string; // author name + content?: string; // extracted article HTML + source?: string; // original publisher domain + published?: string; // publication date string + ttr?: number; // estimated time to read (seconds), 0 = unknown + type?: string; // page type (e.g. "article") } ``` - #### Parameters ##### `input` *required* -URL string links to the article or HTML content of that web page. +URL string or raw HTML content. ##### `parserOptions` *optional* -Object with all or several of the following properties: +| Property | Type | Default | Description | +|---|---|---|---| +| `wordsPerMinute` | `number` | `300` | Words per minute for time-to-read estimation | +| `descriptionTruncateLen` | `number` | `210` | Max characters for generated description | +| `descriptionLengthThreshold` | `number` | `180` | Min characters to keep meta description | +| `contentLengthThreshold` | `number` | `200` | Min characters for article content | +| `allowedTags` | `string[]` | *(semantic/content tags)* | HTML tags to keep in output | +| `allowedAttributes` | `Record` | *(src, href, alt, etc.)* | Per-tag attributes to keep | +| `allowedIframeDomains` | `string[]` | *(youtube, vimeo, etc.)* | Allowed domains for iframe src | - - `wordsPerMinute`: Number, to estimate time to read. Default `300`. - - `descriptionTruncateLen`: Number, max num of chars generated for description. Default `210`. - - `descriptionLengthThreshold`: Number, min num of chars required for description. Default `180`. - - `contentLengthThreshold`: Number, min num of chars required for content. Default `200`. - -For example: - -```js -import { extract } from '@extractus/article-extractor' - -const article = await extract('https://www.cnbc.com/2022/09/21/what-another-major-rate-hike-by-the-federal-reserve-means-to-you.html', { +```ts +const article = await extract(url, { descriptionLengthThreshold: 120, - contentLengthThreshold: 500 -}) - -console.log(article) + contentLengthThreshold: 500, +}); ``` -##### `fetchOptions` *optional* +##### `fetcher` *optional* -`fetchOptions` is an object that can have the following properties: +A custom fetch function with the signature `(url: string) => Promise`. +Use this to customize HTTP behavior: proxy, headers, TLS, authentication, timeouts, etc. -- `headers`: to set request headers -- `proxy`: another endpoint to forward the request to -- `agent`: a HTTP proxy agent -- `signal`: AbortController signal or AbortSignal timeout to terminate the request +Defaults to `globalThis.fetch`. -For example, you can use this param to set request headers to fetch as below: +**Deno** (with proxy): -```js -import { extract } from '@extractus/article-extractor' +```ts +import { extract } from "@extractus/article-extractor"; -const url = 'https://www.cnbc.com/2022/09/21/what-another-major-rate-hike-by-the-federal-reserve-means-to-you.html' -const article = await extract(url, {}, { - headers: { - 'user-agent': 'Opera/9.60 (Windows NT 6.0; U; en) Presto/2.1.1' - } -}) +const client = Deno.createHttpClient({ + proxy: { url: "http://proxy.example.com:8080" }, +}); +const myFetcher = (url: string) => fetch(url, { client }); -console.log(article) +const result = await extract("https://example.com/some-article", {}, myFetcher); ``` -You can also specify a proxy endpoint to load remote content, instead of fetching directly. +**Node.js** (with proxy via undici): -For example: - -```js -import { extract } from '@extractus/article-extractor' +```ts +import { extract } from "@extractus/article-extractor"; +import { fetch, ProxyAgent } from "undici"; -const url = 'https://www.cnbc.com/2022/09/21/what-another-major-rate-hike-by-the-federal-reserve-means-to-you.html' +const dispatcher = new ProxyAgent("http://proxy.example.com:8080"); +const myFetcher = (url: string) => fetch(url, { dispatcher }); -await extract(url, {}, { - headers: { - 'user-agent': 'Opera/9.60 (Windows NT 6.0; U; en) Presto/2.1.1' - }, - proxy: { - target: 'https://your-secret-proxy.io/loadXml?url=', - headers: { - 'Proxy-Authorization': 'Bearer YWxhZGRpbjpvcGVuc2VzYW1l...' - }, - } -}) +const result = await extract("https://example.com/some-article", {}, myFetcher); ``` -Passing requests to proxy is useful while running `@extractus/article-extractor` on browser. View [examples/browser-article-parser](examples/browser-article-parser) as reference example. +**Bun** (with proxy): -For more info about proxy authentication, please refer [HTTP authentication](https://developer.mozilla.org/en-US/docs/Web/HTTP/Authentication) - -For a deeper customization, you can consider using [Proxy](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Proxy) to replace `fetch` behaviors with your own handlers. - -Another way to work with proxy is use `agent` option instead of `proxy` as below: - -```js -import { extract } from '@extractus/article-extractor' - -import { HttpsProxyAgent } from 'https-proxy-agent' - -const proxy = 'http://abc:RaNdoMpasswORd_country-France@proxy.packetstream.io:31113' +```ts +import { extract } from "@extractus/article-extractor"; -const url = 'https://www.cnbc.com/2022/09/21/what-another-major-rate-hike-by-the-federal-reserve-means-to-you.html' +const myFetcher = (url: string) => + fetch(url, { + proxy: "http://proxy.example.com:8080", + }); -const article = await extract(url, {}, { - agent: new HttpsProxyAgent(proxy), -}) -console.log('Run article-extractor with proxy:', proxy) -console.log(article) +const result = await extract("https://example.com/some-article", {}, myFetcher); ``` -For more info about [https-proxy-agent](https://www.npmjs.com/package/https-proxy-agent), check [its repo](https://github.com/TooTallNate/proxy-agents). - -By default, there is no request timeout. You can use the option `signal` to cancel request at the right time. - -The common way is to use AbortControler: - -```js -const controller = new AbortController() - -// stop after 5 seconds -setTimeout(() => { - controller.abort() -}, 5000) - -const data = await extract(url, null, { - signal: controller.signal, -}) -``` +**Custom headers**: -A newer solution is AbortSignal's `timeout()` static method: +```ts +const myFetcher = (url: string) => + fetch(url, { + headers: { + "user-agent": "MyBot/1.0", + authorization: "Bearer token123", + }, + }); -```js -// stop after 5 seconds -const data = await extract(url, null, { - signal: AbortSignal.timeout(5000), -}) +const result = await extract("https://example.com/some-article", {}, myFetcher); ``` -For more info: +**Request timeout**: -- [AbortController constructor](https://developer.mozilla.org/en-US/docs/Web/API/AbortController) -- [AbortSignal: timeout() static method](https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal/timeout_static) +```ts +const myFetcher = (url: string) => + fetch(url, { + signal: AbortSignal.timeout(5000), + }); +const result = await extract("https://example.com/some-article", {}, myFetcher); +``` +--- ### `extractFromHtml()` -Extract article data from HTML string. Return a Promise object as same as `extract()` method above. +Extract article data from an HTML string directly. #### Syntax ```ts -extractFromHtml(String html) -extractFromHtml(String html, String url) -extractFromHtml(String html, String url, Object parserOptions) +extractFromHtml(html: string): Promise +extractFromHtml(html: string, url?: string): Promise +extractFromHtml(html: string, url?: string, parserOptions?: ParserOptions): Promise ``` Example: -```js -import { extractFromHtml } from '@extractus/article-extractor' - -const url = 'https://www.cnbc.com/2022/09/21/what-another-major-rate-hike-by-the-federal-reserve-means-to-you.html' - -const res = await fetch(url) -const html = await res.text() +```ts +import { extractFromHtml } from "jsr:@extractus/article-extractor"; -// you can do whatever with this raw html here: clean up, remove ads banner, etc -// just ensure a html string returned +const res = await fetch(url); +const html = await res.text(); -const article = await extractFromHtml(html, url) -console.log(article) +const article = await extractFromHtml(html, url); ``` #### Parameters ##### `html` *required* -HTML string which contains the article you want to extract. +HTML string containing the article. ##### `url` *optional* -URL string that indicates the source of that HTML content. -`article-extractor` may use this info to handle internal/relative links. +Source URL for resolving relative links. ##### `parserOptions` *optional* See [parserOptions](#parseroptions-optional) above. - --- ### Transformations -Sometimes the default extraction algorithm may not work well. That is the time when we need transformations. +Sometimes the default extraction algorithm may not work well. Transformations let you add pre/post processing per-site. -By adding some functions before and after the main extraction step, we aim to come up with a better result as much as possible. +- `addTransformations(transformation: Transformation | Transformation[]): number` +- `removeTransformations(patterns?: RegExp[]): number` -There are 2 methods to play with transformations: +#### `Transformation` object -- `addTransformations(Object transformation | Array transformations)` -- `removeTransformations(Array patterns)` - -At first, let's talk about `transformation` object. - -#### `transformation` object - -In `@extractus/article-extractor`, `transformation` is an object with the following properties: - -- `patterns`: required, a list of regexps to match the URLs -- `pre`: optional, a function to process raw HTML -- `post`: optional, a function to process extracted article - -Basically, the meaning of `transformation` can be interpreted like this: +```ts +interface Transformation { + patterns: RegExp[]; // URL patterns to match + pre?: (document: Document) => Document; // pre-process raw HTML + post?: (document: Document) => Document; // post-process extracted article +} +``` -> with the urls which match these `patterns`
-> let's run `pre` function to normalize HTML content
-> then extract main article content with normalized HTML, and if success
-> let's run `post` function to normalize extracted article content +> For URLs matching `patterns`, run `pre` on raw HTML, extract article, then run `post` on the result. -![article-extractor extraction process](https://res.cloudinary.com/pwshub/image/upload/v1657336822/documentation/article-parser_extraction_process.png) +![extraction process](https://res.cloudinary.com/pwshub/image/upload/v1657336822/documentation/article-parser_extraction_process.png) -Here is an example transformation: +Example: ```ts -{ - patterns: [ - /([\w]+.)?domain.tld\/*/, - /domain.tld\/articles\/*/ - ], +import { addTransformations } from "jsr:@extractus/article-extractor"; + +addTransformations({ + patterns: [/([\w]+.)?domain\.tld\/*/], pre: (document) => { - // remove all .advertise-area and its siblings from raw HTML content - document.querySelectorAll('.advertise-area').forEach((element) => { - if (element.nodeName === 'DIV') { - while (element.nextSibling) { - element.parentNode.removeChild(element.nextSibling) - } - element.parentNode.removeChild(element) - } - }) - return document + document.querySelectorAll(".advertise-area").forEach((el) => { + el.parentNode?.removeChild(el); + }); + return document; }, post: (document) => { - // with extracted article, replace all h4 tags with h2 - document.querySelectorAll('h4').forEach((element) => { - const h2Element = document.createElement('h2') - h2Element.innerHTML = element.innerHTML - element.parentNode.replaceChild(h2Element, element) - }) - // change small sized images to original version - document.querySelectorAll('img').forEach((element) => { - const src = element.getAttribute('src') - if (src.includes('domain.tld/pics/150x120/')) { - const fullSrc = src.replace('/pics/150x120/', '/pics/original/') - element.setAttribute('src', fullSrc) - } - }) - return document - } -} + document.querySelectorAll("h4").forEach((el) => { + const h2 = document.createElement("h2"); + h2.innerHTML = el.innerHTML; + el.parentNode?.replaceChild(h2, el); + }); + return document; + }, +}); ``` -- To write better transformation logic, please refer [linkedom](https://github.com/WebReflection/linkedom) and [Document Object](https://developer.mozilla.org/en-US/docs/Web/API/Document). +To write better transformations, refer to [linkedom](https://github.com/WebReflection/linkedom) and the [Document API](https://developer.mozilla.org/en-US/docs/Web/API/Document). -#### `addTransformations(Object transformation | Array transformations)` +#### `addTransformations(transformation | Transformation[])` -Add a single transformation or a list of transformations. For example: +Add a single or multiple transformations. Transformations without `patterns` are ignored. ```ts -import { addTransformations } from '@extractus/article-extractor' - -addTransformations({ - patterns: [ - /([\w]+.)?abc.tld\/*/ - ], - pre: (document) => { - // do something with document - return document - }, - post: (document) => { - // do something with document - return document - } -}) +import { addTransformations } from "jsr:@extractus/article-extractor"; addTransformations([ { - patterns: [ - /([\w]+.)?def.tld\/*/ - ], - pre: (document) => { - // do something with document - return document - }, - post: (document) => { - // do something with document - return document - } + patterns: [/([\w]+.)?abc\.tld\/*/], + pre: (doc) => { /* ... */ return doc; }, + post: (doc) => { /* ... */ return doc; }, }, { - patterns: [ - /([\w]+.)?xyz.tld\/*/ - ], - pre: (document) => { - // do something with document - return document - }, - post: (document) => { - // do something with document - return document - } - } -]) -```` - -The transformations without `patterns` will be ignored. - -#### `removeTransformations(Array patterns)` + patterns: [/([\w]+.)?xyz\.tld\/*/], + post: (doc) => { /* ... */ return doc; }, + }, +]); +``` -To remove transformations that match the specific patterns. +#### `removeTransformations(patterns?: RegExp[])` -For example, we can remove all added transformations above: +Remove transformations matching the given patterns. Call without arguments to remove all. -```js -import { removeTransformations } from '@extractus/article-extractor' +```ts +import { removeTransformations } from "jsr:@extractus/article-extractor"; removeTransformations([ - /([\w]+.)?abc.tld\/*/, - /([\w]+.)?def.tld\/*/, - /([\w]+.)?xyz.tld\/*/ -]) + /([\w]+.)?abc\.tld\/*/, + /([\w]+.)?xyz\.tld\/*/, +]); ``` -Calling `removeTransformations()` without parameter will remove all current transformations. - #### Priority order -While processing an article, more than one transformation can be applied. +When multiple transformations match, they all execute in order. -Suppose that we have the following transformations: +Given two transformations matching `goo.gl`: -```ts -[ - { - patterns: [ - /http(s?):\/\/google.com\/*/, - /http(s?):\/\/goo.gl\/*/ - ], - pre: function_one, - post: function_two - }, - { - patterns: [ - /http(s?):\/\/goo.gl\/*/, - /http(s?):\/\/google.inc\/*/ - ], - pre: function_three, - post: function_four - } -] ``` - -As you can see, an article from `goo.gl` certainly matches both them. - -In this scenario, `@extractus/article-extractor` will execute both transformations, one by one: - -`function_one` -> `function_three` -> extraction -> `function_two` -> `function_four` +pre_one -> pre_three -> extraction -> post_two -> post_four +``` --- -### `sanitize-html`'s options +### Content sanitization options -`@extractus/article-extractor` uses [sanitize-html](https://github.com/apostrophecms/sanitize-html) to make a clean sweep of HTML content. +Extracted HTML is sanitized using a built-in DOM tree walker. Disallowed tags are removed (not escaped), and disallowed attributes are stripped. Configure via `parserOptions`: -Here is the [default options](src/config.js#L5) - -Depending on the needs of your content system, you might want to gather some HTML tags/attributes, while ignoring others. - -There are 2 methods to access and modify these options in `@extractus/article-extractor`. - -- `getSanitizeHtmlOptions()` -- `setSanitizeHtmlOptions(Object sanitizeHtmlOptions)` +```ts +import { extract } from "jsr:@extractus/article-extractor"; + +// allow class attributes on and
+const article = await extract(url, { + allowedAttributes: { + a: ["href", "target", "title"], + img: ["src", "srcset", "alt", "title"], + code: ["class"], + div: ["class"], + }, +}); +``` -Read [sanitize-html](https://github.com/apostrophecms/sanitize-html#default-options) docs for more info. +To see the full defaults, refer to [`src/config.ts`](src/config.ts). --- -## Test +## Development ```bash git clone https://github.com/extractus/article-extractor.git cd article-extractor -pnpm i -pnpm test -``` - -![article-extractor-test.png](https://i.imgur.com/TbRCUSS.png?110222) +# run tests +deno test --allow-all -## Quick evaluation +# lint +deno lint -```bash -git clone https://github.com/extractus/article-extractor.git -cd article-extractor -pnpm i -pnpm eval {URL_TO_PARSE_ARTICLE} +# build npm package +deno run -A ./scripts/build_npm.ts ``` ## License @@ -499,12 +369,12 @@ The MIT License (MIT) ## Support the project -If you find value from this open source project, you can support in the following ways: +This project is maintained in my spare time. If you find it helpful, there are a few simple ways to support its continued development: -- Give it a star โญ -- Buy me a coffee: https://paypal.me/ndaidong ๐Ÿต -- Subscribe [Article Extractor service](https://rapidapi.com/pwshub-pwshub-default/api/article-extractor2) on RapidAPI ๐Ÿ˜‰ +* โญ Star this repository to help more people discover it. +* โ˜• Buy me a coffee: https://paypal.me/ndaidong +* ๐Ÿš€ Subscribe to the [Article Extractor service](https://rapidapi.com/pwshub-pwshub-default/api/article-extractor2) on RapidAPI. -Thank you. +Every bit of support helps keep this project actively maintained. Thank you! โค๏ธ --- diff --git a/SECURITY.md b/SECURITY.md index 921d17aa..729e7272 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -12,6 +12,6 @@ Description above is a general rule and may be altered on case by case basis. You can report low severity vulnerabilities as GitHub issues. -More severe vulnerabilities should be reported to email extractus@pwshub.com. +More severe vulnerabilities should be reported to email extractus.security@aslot.win. --- diff --git a/deno.json b/deno.json new file mode 100644 index 00000000..cb0a6abe --- /dev/null +++ b/deno.json @@ -0,0 +1,47 @@ +{ + "name": "@extractus/article-extractor", + "version": "9.0.0", + "description": "To extract main article from given URL", + "homepage": "https://github.com/extractus/article-extractor", + "repository": { + "type": "git", + "url": "git@github.com:extractus/article-extractor.git" + }, + "author": "@extractus", + "license": "MIT", + "exports": "./mod.ts", + "tasks": { + "test": "deno test --allow-all", + "lint": "deno lint", + "build": "deno run -A ./scripts/build_npm.ts" + }, + "imports": { + "@deno/dnt": "jsr:@deno/dnt@^0.43.2", + "@pwshub/bellajs": "jsr:@pwshub/bellajs@13.1.0", + "@std/assert": "jsr:@std/assert@1.0.12", + "@mozilla/readability": "npm:@mozilla/readability@^0.6.0", + "linkedom": "npm:linkedom@^0.18.13" + }, + "compilerOptions": { + "lib": ["dom", "dom.iterable", "dom.asynciterable", "deno.ns"] + }, + "lint": { + "include": ["src/", "mod.ts", "scripts/", "tests/"], + "exclude": ["npm/"], + "rules": { + "tags": ["recommended"], + "exclude": ["no-explicit-any", "no-control-regex"] + } + }, + "fmt": { + "include": ["src/", "mod.ts", "scripts/", "tests/"], + "exclude": ["npm/"] + }, + "test": { + "include": ["tests/"] + }, + "publish": { + "include": ["LICENSE", "README.md", "mod.ts", "src/"], + "exclude": ["tests/", "scripts/"] + } +} diff --git a/deno.lock b/deno.lock new file mode 100644 index 00000000..8ea63eb3 --- /dev/null +++ b/deno.lock @@ -0,0 +1,198 @@ +{ + "version": "5", + "specifiers": { + "jsr:@david/code-block-writer@^13.0.3": "13.0.3", + "jsr:@deno/dnt@~0.43.2": "0.43.2", + "jsr:@pwshub/bellajs@13.1.0": "13.1.0", + "jsr:@std/assert@1.0.12": "1.0.12", + "jsr:@std/fmt@1": "1.0.10", + "jsr:@std/fs@1": "1.0.24", + "jsr:@std/internal@^1.0.14": "1.0.14", + "jsr:@std/internal@^1.0.6": "1.0.14", + "jsr:@std/path@1": "1.1.6", + "jsr:@std/path@^1.1.5": "1.1.6", + "jsr:@ts-morph/bootstrap@0.29": "0.29.0", + "jsr:@ts-morph/common@0.29": "0.29.0", + "npm:@mozilla/readability@0.6": "0.6.0", + "npm:linkedom@~0.18.13": "0.18.13" + }, + "jsr": { + "@david/code-block-writer@13.0.3": { + "integrity": "f98c77d320f5957899a61bfb7a9bead7c6d83ad1515daee92dbacc861e13bb7f" + }, + "@deno/dnt@0.43.2": { + "integrity": "8068703a782f389d9fcb4ead32809ca666d3b088262a23f1d9016cbc9af5d37f", + "dependencies": [ + "jsr:@david/code-block-writer", + "jsr:@std/fmt", + "jsr:@std/fs", + "jsr:@std/path@1", + "jsr:@ts-morph/bootstrap" + ] + }, + "@pwshub/bellajs@13.1.0": { + "integrity": "3b2e3074c621edbab0514cd1737a1564765deefe55237f56663c54675cb5d315" + }, + "@std/assert@1.0.12": { + "integrity": "08009f0926dda9cbd8bef3a35d3b6a4b964b0ab5c3e140a4e0351fbf34af5b9a", + "dependencies": [ + "jsr:@std/internal@^1.0.6" + ] + }, + "@std/fmt@1.0.10": { + "integrity": "90dfba288802ac6de82fb31d0917eb9e4450b9925b954d5e51fc29ac07419db5" + }, + "@std/fs@1.0.24": { + "integrity": "f3061b45b81673a2bece689da041df32d174be064c89eb6397fb5718d3fb7877", + "dependencies": [ + "jsr:@std/internal@^1.0.14", + "jsr:@std/path@^1.1.5" + ] + }, + "@std/internal@1.0.14": { + "integrity": "291516b3d4c35024d6ffbc0a9df5bf4c64116e05b50012cf846710152d2ffdf7" + }, + "@std/path@1.1.6": { + "integrity": "c68485c2a4dfbb5ae3cc74fae4e8c4e5d874cf8a8ed12927917235c758b46cbe", + "dependencies": [ + "jsr:@std/internal@^1.0.14" + ] + }, + "@ts-morph/bootstrap@0.29.0": { + "integrity": "69f9b1b42b03423308a9f623f924174181ef20f6f2a8771799f216f561b9b7a6", + "dependencies": [ + "jsr:@ts-morph/common" + ] + }, + "@ts-morph/common@0.29.0": { + "integrity": "14e8f44c5e4297628cf756aed071787dc026c493e36c1cc29ce49dc95e493f8f", + "dependencies": [ + "jsr:@std/fs", + "jsr:@std/path@1" + ] + } + }, + "npm": { + "@mozilla/readability@0.6.0": { + "integrity": "sha512-juG5VWh4qAivzTAeMzvY9xs9HY5rAcr2E4I7tiSSCokRFi7XIZCAu92ZkSTsIj1OPceCifL3cpfteP3pDT9/QQ==" + }, + "boolbase@2.0.0": { + "integrity": "sha512-DkVaaQHymRhpYEYo9x1oo7Q7B0Y6KJUsjm3c9eTyFDby4MHLBTwZ6ZDWBel5zrYxj1WsZgC5oLpiz+93MluXeA==" + }, + "css-select@7.0.0": { + "integrity": "sha512-snmjEVXy+1LnwXdxhYvTMj1d9tOh4HxkA1YmoayVBeeyR2C14Pum7fcxJIm4SswYspVy866eYNwlH6xC3/VH5g==", + "dependencies": [ + "boolbase", + "css-what", + "domhandler@6.0.1", + "domutils@4.0.2", + "nth-check" + ] + }, + "css-what@8.0.0": { + "integrity": "sha512-DH0Bqq3DNp5tdOReuNyAA+Ev4Y2GS5FMbZpeTLP6C4CDi0h5nL0BmUPChXw3o/qbHLDWHl49sbNqQVY7bMSDdw==" + }, + "cssom@0.5.0": { + "integrity": "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw==" + }, + "dom-serializer@2.0.0": { + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "dependencies": [ + "domelementtype@2.3.0", + "domhandler@5.0.3", + "entities@4.5.0" + ] + }, + "dom-serializer@3.1.1": { + "integrity": "sha512-4MEa38/QexBob6gFNwu+EGdWvhJ1OKuNwdYY3Y3NyeWDQfnGeDYQUDfIRzWu5B5gsv03so2Uxd28YC6zrsx3Lw==", + "dependencies": [ + "domelementtype@3.0.0", + "domhandler@6.0.1", + "entities@8.0.0" + ] + }, + "domelementtype@2.3.0": { + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==" + }, + "domelementtype@3.0.0": { + "integrity": "sha512-umCQid3jKbDmVjx8jGaW7uUykm4DEUeyV21hPxNMo2nV955DhUThwqyOIDtreepP31hl84X7G5U9ZfsWvIB3Pg==" + }, + "domhandler@5.0.3": { + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "dependencies": [ + "domelementtype@2.3.0" + ] + }, + "domhandler@6.0.1": { + "integrity": "sha512-gYzvtM72ZtxQO0T048kd6HWSbbGCNOUwcnfQ01cqIJ4X2IYKFFHZ5mKvrQETcFXxsRObZulDaKmy//R7TPtsBg==", + "dependencies": [ + "domelementtype@3.0.0" + ] + }, + "domutils@3.2.2": { + "integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==", + "dependencies": [ + "dom-serializer@2.0.0", + "domelementtype@2.3.0", + "domhandler@5.0.3" + ] + }, + "domutils@4.0.2": { + "integrity": "sha512-qI4JLRKnSzqFqr7hAlS5xQDusBCjKSEG4t4+7aNrIQMHBcsC2TGEhuyABJdYkgSewL57PNLYEiibY2iPKhKpaA==", + "dependencies": [ + "dom-serializer@3.1.1", + "domelementtype@3.0.0", + "domhandler@6.0.1" + ] + }, + "entities@4.5.0": { + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==" + }, + "entities@7.0.1": { + "integrity": "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==" + }, + "entities@8.0.0": { + "integrity": "sha512-zwfzJecQ/Uej6tusMqwAqU/6KL2XaB2VZ2Jg54Je6ahNBGNH6Ek6g3jjNCF0fG9EWQKGZNddNjU5F1ZQn/sBnA==" + }, + "html-escaper@3.0.3": { + "integrity": "sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ==" + }, + "htmlparser2@10.1.0": { + "integrity": "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==", + "dependencies": [ + "domelementtype@2.3.0", + "domhandler@5.0.3", + "domutils@3.2.2", + "entities@7.0.1" + ] + }, + "linkedom@0.18.13": { + "integrity": "sha512-ES/o9qotMpzpN2MHs+Iq/JcVoOj8Fa5wiQYrTdFpvAnwXL0g66XHHUc9WUMk6nAlBtGsFQ24ne+SYnvnaQ2FSw==", + "dependencies": [ + "css-select", + "cssom", + "html-escaper", + "htmlparser2", + "uhyphen" + ] + }, + "nth-check@3.0.1": { + "integrity": "sha512-GX0gsdbGVCgnRgbeGaubfjpBXyYRWOOCVeYh08bSQvDZqxz5ndXs1OTfAt/h36G1xvI94YIspsI0sVFqAV9+RQ==", + "dependencies": [ + "boolbase" + ] + }, + "uhyphen@0.2.0": { + "integrity": "sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA==" + } + }, + "workspace": { + "dependencies": [ + "jsr:@deno/dnt@~0.43.2", + "jsr:@pwshub/bellajs@13.1.0", + "jsr:@std/assert@1.0.12", + "npm:@mozilla/readability@0.6", + "npm:linkedom@~0.18.13" + ] + } +} diff --git a/eslint.config.js b/eslint.config.js deleted file mode 100644 index 6701eb1e..00000000 --- a/eslint.config.js +++ /dev/null @@ -1,127 +0,0 @@ -// eslint.config.js - -import eslintjs from '@eslint/js' -import globals from 'globals' - -export default [ - eslintjs.configs.recommended, - { - languageOptions: { - ecmaVersion: 'latest', - sourceType: 'module', - globals: { - ...globals.node, - ...globals.browser, - Intl: 'readonly', - }, - }, - ignores: [ - 'node_modules', - 'storage', - ], - rules: { - 'arrow-spacing': ['error', { 'before': true, 'after': true }], - 'block-spacing': ['error', 'always'], - 'brace-style': ['error', '1tbs', { 'allowSingleLine': true }], - 'camelcase': ['error', { - 'allow': ['^UNSAFE_'], - 'properties': 'never', - 'ignoreGlobals': true, - }], - 'comma-dangle': ['error', { - 'arrays': 'always-multiline', - 'objects': 'always-multiline', - 'imports': 'never', - 'exports': 'never', - 'functions': 'never', - }], - 'comma-spacing': ['error', { 'before': false, 'after': true }], - 'eol-last': 'error', - 'eqeqeq': ['error', 'always', { 'null': 'ignore' }], - 'func-call-spacing': ['error', 'never'], - 'indent': [ - 'error', - 2, - { - 'MemberExpression': 1, - 'FunctionDeclaration': { - 'body': 1, - 'parameters': 2, - }, - 'SwitchCase': 1, - 'ignoredNodes': ['TemplateLiteral > *'], - }, - ], - 'key-spacing': ['error', { 'beforeColon': false, 'afterColon': true }], - 'keyword-spacing': ['error', { 'before': true, 'after': true }], - 'lines-between-class-members': ['error', 'always', { 'exceptAfterSingleLine': true }], - 'max-len': [ - 'error', - { - 'code': 120, - 'ignoreTrailingComments': true, - 'ignoreComments': true, - 'ignoreUrls': true, - }, - ], - 'max-lines': [ - 'error', - { - 'max': 360, - 'skipBlankLines': true, - 'skipComments': false, - }, - ], - 'max-lines-per-function': [ - 'error', - { - 'max': 180, - 'skipBlankLines': true, - }, - ], - 'max-params': ['error', 3], - 'no-array-constructor': 'error', - 'no-mixed-spaces-and-tabs': 'error', - 'no-multi-spaces': 'error', - 'no-multi-str': 'error', - 'no-multiple-empty-lines': [ - 'error', - { - 'max': 1, - 'maxEOF': 0, - }, - ], - 'no-restricted-syntax': [ - 'error', - 'WithStatement', - 'BinaryExpression[operator=\'in\']', - ], - 'no-trailing-spaces': 'error', - 'no-use-before-define': [ - 'error', - { - 'functions': true, - 'classes': true, - 'variables': false, - }, - ], - 'no-var': 'warn', - 'object-curly-spacing': ['error', 'always'], - 'padded-blocks': [ - 'error', - { - 'blocks': 'never', - 'switches': 'never', - 'classes': 'never', - }, - ], - 'quotes': ['error', 'single'], - 'space-before-blocks': ['error', 'always'], - 'space-before-function-paren': ['error', 'always'], - 'space-infix-ops': 'error', - 'space-unary-ops': ['error', { 'words': true, 'nonwords': false }], - 'space-in-parens': ['error', 'never'], - 'semi': ['error', 'never'], - }, - }, -] diff --git a/eval.js b/eval.js deleted file mode 100644 index 993869b1..00000000 --- a/eval.js +++ /dev/null @@ -1,53 +0,0 @@ -// eval.js - -import { execSync } from 'node:child_process' -import { readFileSync, writeFileSync, existsSync } from 'node:fs' - -import { slugify } from '@ndaidong/bellajs' - -import { isValid as isValidUrl } from './src/utils/linker.js' -import { extract, extractFromHtml } from './src/main.js' - -if (!existsSync('evaluation')) { - execSync('mkdir evaluation') -} - -const extractFromUrl = async (url) => { - try { - console.time('extraction') - const art = await extract(url) - console.log(art) - if (art) { - const slug = slugify(art.title) - writeFileSync(`evaluation/${slug}.html`, art.content, 'utf8') - } - console.timeEnd('extraction') - } catch (err) { - console.trace(err) - } -} - -const extractFromFile = async (fpath) => { - try { - const html = readFileSync(fpath, 'utf8') - const art = await extractFromHtml(html) - console.log(art) - if (art) { - const slug = slugify(art.title) - writeFileSync(`evaluation/${slug}.html`, art.content, 'utf8') - } - } catch (err) { - console.trace(err) - } -} - -const init = (argv) => { - if (argv.length === 3) { - const input = argv[2] - const isUrl = isValidUrl(input) - return isUrl ? extractFromUrl(input) : existsSync(input) ? extractFromFile(input) : false - } - return 'Nothing to do!' -} - -init(process.argv) diff --git a/index.d.ts b/index.d.ts deleted file mode 100644 index 9cb82521..00000000 --- a/index.d.ts +++ /dev/null @@ -1,140 +0,0 @@ -// Type definitions - -import { IOptions as SanitizeOptions } from 'sanitize-html' - -/** - * Transformation for per-site HTML pre/post processing. - */ -export interface Transformation { - /** URL regex patterns to match */ - patterns: Array - /** Function to pre-process raw HTML before extraction */ - pre?: (document: Document) => Document - /** Function to post-process extracted article content */ - post?: (document: Document) => Document -} - -/** - * Options for the article extraction process. - */ -export interface ParserOptions { - /** Words per minute for time-to-read estimation. Default: 300 */ - wordsPerMinute?: number - /** Max chars for generated description. Default: 210 */ - descriptionTruncateLen?: number - /** Min chars required for description. Default: 180 */ - descriptionLengthThreshold?: number - /** Min chars required for content. Default: 200 */ - contentLengthThreshold?: number -} - -/** - * Proxy configuration for fetching articles. - */ -export interface ProxyConfig { - /** Proxy endpoint URL */ - target?: string - /** Headers for proxy request */ - headers?: Record -} - -/** - * Options for the HTTP fetch request. - */ -export interface FetchOptions { - /** Custom request headers */ - headers?: Record - /** Proxy configuration */ - proxy?: ProxyConfig - /** HTTP proxy agent (e.g. HttpsProxyAgent) */ - agent?: object - /** AbortSignal to cancel the request */ - signal?: object -} - -/** - * Extracted article data structure. - */ -export interface ArticleData { - /** Best resolved URL of the article */ - url?: string - /** Alternative URLs (canonical, shortlink, etc.) */ - links?: string[] - /** Article title */ - title?: string - /** Short description or excerpt */ - description?: string - /** Main image URL */ - image?: string - /** Site favicon URL */ - favicon?: string - /** Author name */ - author?: string - /** Extracted article HTML content */ - content?: string - /** Original publisher/source domain */ - source?: string - /** Publication date string */ - published?: string - /** Estimated time to read in seconds (0 = unknown) */ - ttr?: number - /** Page type (e.g. article) */ - type?: string -} - -/** - * Register one or more transformations for per-site HTML processing. - * - * @param transformations - Single transformation or array of transformations - * @returns Number of transformations successfully added - */ -export function addTransformations (transformations: Transformation | Array): number - -/** - * Remove transformations matching the given patterns. - * Calling without arguments removes all transformations. - * - * @param patterns - URL patterns to match for removal - * @returns Number of transformations removed - */ -export function removeTransformations (patterns?: Array): number - -/** - * Get a copy of the current sanitize-html options. - */ -export function getSanitizeHtmlOptions (): SanitizeOptions - -/** - * Update sanitize-html options by merging with the current ones. - * - * @param options - Partial sanitize options to merge - */ -export function setSanitizeHtmlOptions (options: SanitizeOptions): void - -/** - * Load and extract article data from a URL or HTML string. - * - * @param input - URL or HTML string to extract from - * @param parserOptions - Options for parsing - * @param fetchOptions - Options for HTTP fetch - * @returns Extracted article data or null - */ -export function extract ( - input: string, - parserOptions?: ParserOptions, - fetchOptions?: FetchOptions, -): Promise - -/** - * Extract article data from an HTML string directly. - * - * @param html - Raw HTML content - * @param url - Source URL for resolving relative links - * @param parserOptions - Options for parsing - * @returns Extracted article data or null - */ -export function extractFromHtml ( - html: string, - url?: string, - parserOptions?: ParserOptions, -): Promise diff --git a/mod.ts b/mod.ts new file mode 100644 index 00000000..6842e096 --- /dev/null +++ b/mod.ts @@ -0,0 +1,12 @@ +export { + addTransformations, + extract, + extractFromHtml, + removeTransformations, +} from "./src/main.ts"; +export type { + ArticleData, + Fetcher, + ParserOptions, + Transformation, +} from "./src/main.ts"; diff --git a/package.json b/package.json deleted file mode 100644 index 14fb9893..00000000 --- a/package.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "version": "8.1.0", - "name": "@extractus/article-extractor", - "description": "To extract main article from given URL", - "homepage": "https://github.com/extractus/article-extractor", - "repository": { - "type": "git", - "url": "git@github.com:extractus/article-extractor.git" - }, - "author": "@extractus", - "main": "./src/main.js", - "type": "module", - "exports": { - ".": { - "types": "./index.d.ts", - "import": "./src/main.js", - "default": "./src/main.js" - } - }, - "types": "./index.d.ts", - "engines": { - "node": ">= 20" - }, - "scripts": { - "lint": "eslint .", - "lint:fix": "eslint --fix .", - "pretest": "npm run lint", - "test": "node --test --experimental-test-coverage", - "eval": "node eval", - "reset": "node reset" - }, - "dependencies": { - "@mozilla/readability": "^0.6.0", - "@pwshub/bellajs": "^13.0.2", - "linkedom": "^0.18.12", - "sanitize-html": "^2.17.3" - }, - "devDependencies": { - "@eslint/js": "^10.0.1", - "@types/sanitize-html": "^2.16.1", - "eslint": "^10.3.0", - "globals": "^17.6.0", - "https-proxy-agent": "^9.0.0", - "nock": "^14.0.14" - }, - "files": [ - "src", - "index.d.ts" - ], - "sideEffects": false, - "keywords": [ - "article", - "extractor", - "parser", - "readability", - "util" - ], - "license": "MIT" -} diff --git a/reset.js b/reset.js deleted file mode 100644 index 6afa5a09..00000000 --- a/reset.js +++ /dev/null @@ -1,34 +0,0 @@ -// reset.js - -import { - existsSync, - unlinkSync -} from 'node:fs' - -import { execSync } from 'node:child_process' - -const dirs = [ - 'evaluation', - 'docs', - '.nyc_output', - 'coverage', - 'node_modules', - '.nuxt', -] - -const files = [ - 'yarn.lock', - 'pnpm-lock.yaml', - 'package-lock.json', - 'coverage.lcov', -] - -dirs.forEach((d) => { - execSync(`rm -rf ${d}`) -}) - -files.forEach((f) => { - if (existsSync(f)) { - unlinkSync(f) - } -}) diff --git a/scripts/build_npm.ts b/scripts/build_npm.ts new file mode 100644 index 00000000..cdd47856 --- /dev/null +++ b/scripts/build_npm.ts @@ -0,0 +1,39 @@ +// scripts/build_npm.ts + +import { build, emptyDir } from "@deno/dnt"; +import pkg from "../deno.json" with { type: "json" }; + +const outputDir = "./npm"; + +await emptyDir(outputDir); + +await build({ + entryPoints: ["./mod.ts"], + outDir: outputDir, + shims: { + deno: true, + }, + test: false, + typeCheck: false, + tsconfig: { + compilerOptions: { + target: "es2022", + lib: ["es2022", "dom"], + }, + }, + package: { + name: pkg.name, + version: pkg.version, + description: pkg.description, + author: pkg.author, + repository: pkg.repository, + bugs: { + url: `${pkg.homepage}/issues`, + }, + license: pkg.license, + }, + postBuild() { + Deno.copyFileSync("LICENSE", "npm/LICENSE"); + Deno.copyFileSync("README.md", "npm/README.md"); + }, +}); diff --git a/src/config.js b/src/config.js deleted file mode 100644 index 1554625f..00000000 --- a/src/config.js +++ /dev/null @@ -1,80 +0,0 @@ -// config.js - -import { clone } from '@pwshub/bellajs' - -/** - * Default sanitize-html options for cleaning extracted article content. - * Defines allowed HTML tags, attributes, and iframe domains. - * - * @type {SanitizeOptions} - */ -const sanitizeHtmlOptions = { - allowedTags: [ - 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', - 'u', 'b', 'i', 'em', 'strong', 'small', 'sup', 'sub', - 'div', 'span', 'p', 'article', 'blockquote', 'section', - 'details', 'summary', - 'pre', 'code', - 'ul', 'ol', 'li', 'dd', 'dl', - 'table', 'th', 'tr', 'td', 'thead', 'tbody', 'tfoot', - 'fieldset', 'legend', - 'figure', 'figcaption', 'img', 'picture', - 'video', 'audio', 'source', - 'iframe', - 'progress', - 'br', 'p', 'hr', - 'label', - 'abbr', - 'a', - 'svg', - ], - allowedAttributes: { - h1: ['id'], - h2: ['id'], - h3: ['id'], - h4: ['id'], - h5: ['id'], - h6: ['id'], - a: ['href', 'target', 'title'], - abbr: ['title'], - progress: ['value', 'max'], - img: ['src', 'srcset', 'alt', 'title'], - picture: ['media', 'srcset'], - video: ['controls', 'width', 'height', 'autoplay', 'muted', 'loop', 'src'], - audio: ['controls', 'width', 'height', 'autoplay', 'muted', 'loop', 'src'], - source: ['src', 'srcset', 'data-srcset', 'type', 'media', 'sizes'], - iframe: ['src', 'frameborder', 'height', 'width', 'scrolling', 'allow'], - svg: ['width', 'height'], // sanitize-html does not support svg fully yet - }, - allowedIframeDomains: [ - 'youtube.com', 'vimeo.com', 'odysee.com', - 'soundcloud.com', 'audius.co', - 'github.com', 'codepen.com', - 'twitter.com', 'facebook.com', 'instagram.com', - ], - disallowedTagsMode: 'discard', - allowVulnerableTags: false, - parseStyleAttributes: false, - enforceHtmlBoundary: false, -} - -/** - * Get a clone of the current sanitize-html options. - * - * @returns {SanitizeOptions} Cloned sanitize options - */ -export const getSanitizeHtmlOptions = () => { - return clone(sanitizeHtmlOptions) -} - -/** - * Update sanitize-html options by merging with the current ones. - * - * @param {SanitizeOptions} [opts={}] - Partial options to merge - * @returns {void} - */ -export const setSanitizeHtmlOptions = (opts = {}) => { - Object.keys(opts).forEach((key) => { - sanitizeHtmlOptions[key] = clone(opts[key]) - }) -} diff --git a/src/config.test.js b/src/config.test.js deleted file mode 100644 index cf3ba632..00000000 --- a/src/config.test.js +++ /dev/null @@ -1,37 +0,0 @@ -// config.test -import { describe, it } from 'node:test' -import assert from 'node:assert' - -import { - setSanitizeHtmlOptions, - getSanitizeHtmlOptions -} from './config.js' - -describe('check config methods', () => { - it('Testing setSanitizeHtmlOptions/getSanitizeHtmlOptions methods', () => { - setSanitizeHtmlOptions({ - allowedTags: ['div', 'span'], - allowedAttributes: { - a: ['href', 'title'], - }, - }) - - const actual = getSanitizeHtmlOptions() - const actualAllowedAttributes = actual.allowedAttributes - const expectedAllowedAttributes = { - a: ['href', 'title'], - } - - assert.deepEqual(actualAllowedAttributes, expectedAllowedAttributes) - - const actualAllowedTags = actual.allowedTags - const expectedAllowedTags = ['div', 'span'] - assert.deepEqual(actualAllowedTags, expectedAllowedTags) - - setSanitizeHtmlOptions({ - allowedTags: [], - }) - - assert.deepEqual(getSanitizeHtmlOptions().allowedTags, []) - }) -}) diff --git a/src/config.ts b/src/config.ts new file mode 100644 index 00000000..1504747b --- /dev/null +++ b/src/config.ts @@ -0,0 +1,89 @@ +// config.ts + +export const defaultAllowedTags: string[] = [ + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "u", + "b", + "i", + "em", + "strong", + "small", + "sup", + "sub", + "div", + "span", + "p", + "article", + "blockquote", + "section", + "details", + "summary", + "pre", + "code", + "ul", + "ol", + "li", + "dd", + "dl", + "table", + "th", + "tr", + "td", + "thead", + "tbody", + "tfoot", + "fieldset", + "legend", + "figure", + "figcaption", + "img", + "picture", + "video", + "audio", + "source", + "iframe", + "progress", + "br", + "hr", + "label", + "abbr", + "a", + "svg", +]; + +export const defaultAllowedAttributes: Record = { + h1: ["id"], + h2: ["id"], + h3: ["id"], + h4: ["id"], + h5: ["id"], + h6: ["id"], + a: ["href", "target", "title"], + abbr: ["title"], + progress: ["value", "max"], + img: ["src", "srcset", "alt", "title"], + picture: ["media", "srcset"], + video: ["controls", "width", "height", "autoplay", "muted", "loop", "src"], + audio: ["controls", "width", "height", "autoplay", "muted", "loop", "src"], + source: ["src", "srcset", "data-srcset", "type", "media", "sizes"], + iframe: ["src", "frameborder", "height", "width", "scrolling", "allow"], + svg: ["width", "height"], +}; + +export const defaultAllowedIframeDomains: string[] = [ + "youtube.com", + "vimeo.com", + "odysee.com", + "soundcloud.com", + "audius.co", + "github.com", + "codepen.com", + "twitter.com", + "facebook.com", + "instagram.com", +]; diff --git a/src/main.js b/src/main.js deleted file mode 100644 index bffaf4a7..00000000 --- a/src/main.js +++ /dev/null @@ -1,52 +0,0 @@ -// main.js - -import { - isString -} from '@pwshub/bellajs' - -import retrieve from './utils/retrieve.js' -import parseFromHtml from './utils/parseFromHtml.js' -import { getCharset } from './utils/html.js' -import { isValid as isValidUrl } from './utils/linker.js' - -/** - * Load and extract article data from a URL or HTML string. - * - * @param {string} input - URL or HTML string to extract from - * @param {ParserOptions} [parserOptions={}] - Options for parsing - * @param {FetchOptions} [fetchOptions={}] - Options for HTTP fetch - * @returns {Promise} Extracted article data or null - */ -export const extract = async (input, parserOptions = {}, fetchOptions = {}) => { - if (!isString(input)) { - throw new Error('Input must be a string') - } - - if (!isValidUrl(input)) { - return parseFromHtml(input, null, parserOptions) - } - const buffer = await retrieve(input, fetchOptions) - const text = buffer ? Buffer.from(buffer).toString().trim() : '' - if (!text) { - return null - } - const charset = getCharset(text) - const decoder = new TextDecoder(charset) - const html = decoder.decode(buffer) - return parseFromHtml(html, input, parserOptions) -} - -/** - * Extract article data from an HTML string directly. - * - * @param {string} html - Raw HTML content - * @param {string} [url] - Source URL for resolving relative links - * @param {ParserOptions} [parserOptions={}] - Options for parsing - * @returns {Promise} Extracted article data or null - */ -export const extractFromHtml = async (html, url, parserOptions = {}) => { - return parseFromHtml(html, url, parserOptions) -} - -export { addTransformations, removeTransformations } from './utils/transformation.js' -export { setSanitizeHtmlOptions, getSanitizeHtmlOptions } from './config.js' diff --git a/src/main.test.js b/src/main.test.js deleted file mode 100644 index b52794f2..00000000 --- a/src/main.test.js +++ /dev/null @@ -1,164 +0,0 @@ -// main.test - -import { describe, it } from 'node:test' -import assert from 'node:assert' - -import { readFileSync } from 'fs' - -import { HttpsProxyAgent } from 'https-proxy-agent' - -import nock from 'nock' - -import { - extract, - getSanitizeHtmlOptions, - setSanitizeHtmlOptions, - addTransformations, - removeTransformations -} from './main.js' - -const env = process.env || {} -const PROXY_SERVER = env.PROXY_SERVER || '' - -const parseUrl = (url) => { - const re = new URL(url) - return { - baseUrl: `${re.protocol}//${re.host}`, - path: re.pathname, - } -} - -describe('check all exported methods', () => { - const fns = [ - extract, - getSanitizeHtmlOptions, - setSanitizeHtmlOptions, - addTransformations, - removeTransformations, - ] - - fns.forEach((fn) => { - it(` check ${fn.name}`, () => { - assert.ok(fn) - }) - }) -}) - -describe('test extract(bad url)', () => { - const badSamples = [ - '', - { k: 9 }, - [1, 3, 4], - 301932, - 'htt:/abc.com/failed-none-sense', - 'fpt://abc.com/failed-none-sense', - 'ttp://badcom/146753785', - 'https://674458092126388225', - 'https://soundcloud^(*%%$%^$$%$$*&(&)())', - ] - - badSamples.forEach((url) => { - it(`testing extract bad url "${url}"`, async () => { - try { - await extract(url) - } catch (err) { - assert.ok(err) - } - }) - }) -}) - -describe('test extract(regular article url)', () => { - const expDesc = [ - 'Navigation here Few can name a rational peach that isn\'t a conscientious goldfish!', - 'One cannot separate snakes from plucky pomegranates?', - 'Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs.', - ].join(' ') - const cases = [ - { - input: { - url: 'https://somewhere.com/path/to/no/article', - html: readFileSync('./test-data/html-no-article.html', 'utf8'), - }, - validate: (result) => { - assert.equal(result, null) - }, - }, - { - input: { - url: 'https://somewhere.com/path/to/no/content', - html: '', - }, - validate: (result) => { - assert.equal(result, null) - }, - }, - { - input: { - url: 'https://somewhere.com/path/to/article', - html: readFileSync('./test-data/regular-article.html', 'utf8'), - }, - validate: (result) => { - assert.ok(result) - assert.equal(result.title, 'Article title here') - assert.equal(result.description, expDesc) - }, - }, - ] - cases.forEach(({ input, validate }) => { - const { url, html, statusCode = 200 } = input - const { baseUrl, path } = parseUrl(url) - const scope = nock(baseUrl) - scope.get(path) - .reply(statusCode, html, { - 'Content-Type': 'text/html', - }) - it(`check extract("${url}")`, async () => { - const result = await extract(url) - validate(result) - }) - }) - - it('check extract(html string)', async () => { - const html = readFileSync('./test-data/regular-article.html', 'utf8') - const result = await extract(html) - assert.ok(result) - assert.equal(result.title, 'Article title here') - assert.equal(result.description, expDesc) - }) -}) - -describe('test extract with modified sanitize-html options', () => { - const currentSanitizeOptions = getSanitizeHtmlOptions() - - setSanitizeHtmlOptions({ - ...currentSanitizeOptions, - allowedAttributes: { - ...currentSanitizeOptions.allowedAttributes, - code: ['class'], - div: ['class'], - }, - allowedClasses: { - code: ['language-*', 'lang-*'], - }, - }) - - it('check if output contain class attribute', async () => { - const html = readFileSync('./test-data/article-with-classes-attributes.html', 'utf8') - const result = await extract(html) - assert.ok(result.content.includes('code class="lang-js"')) - }) -}) - -if (PROXY_SERVER !== '') { - describe('test extract live article API via proxy server', () => { - it('check if extract method works with proxy server', async () => { - const url = 'https://www.cnbc.com/2022/09/21/what-another-major-rate-hike-by-the-federal-reserve-means-to-you.html' - const result = await extract(url, {}, { - agent: new HttpsProxyAgent(PROXY_SERVER), - }) - assert.ok(result.title.includes('Federal Reserve')) - assert.equal(result.source, 'cnbc.com') - }, 10000) - }) -} diff --git a/src/main.ts b/src/main.ts new file mode 100644 index 00000000..06e3d34a --- /dev/null +++ b/src/main.ts @@ -0,0 +1,120 @@ +// main.ts + +import { isString } from "@pwshub/bellajs"; + +import retrieve from "./utils/retrieve.ts"; +import parseFromHtml from "./utils/parseFromHtml.ts"; +import { getCharset } from "./utils/html.ts"; +import { isValid as isValidUrl } from "./utils/linker.ts"; + +/** Custom fetch function signature. */ +export type Fetcher = (url: string) => Promise; + +/** Per-site HTML pre/post processing transformation. */ +export interface Transformation { + /** URL regex patterns to match against article links. */ + patterns: RegExp[]; + /** Function to pre-process raw HTML before extraction. */ + pre?: (document: Document) => Document; + /** Function to post-process extracted article content. */ + post?: (document: Document) => Document; +} + +/** Options for the article extraction process. */ +export interface ParserOptions { + /** Words per minute for time-to-read estimation. Default: 300 */ + wordsPerMinute?: number; + /** Max characters for generated description. Default: 210 */ + descriptionTruncateLen?: number; + /** Min characters required to keep meta description. Default: 180 */ + descriptionLengthThreshold?: number; + /** Min characters required for article content. Default: 200 */ + contentLengthThreshold?: number; + /** Allowed HTML tags in output. Default: list of semantic/content tags. */ + allowedTags?: string[]; + /** Per-tag allowed attributes. Default: src, href, alt, etc. */ + allowedAttributes?: Record; + /** Allowed domains for iframe src. Default: youtube, vimeo, etc. */ + allowedIframeDomains?: string[]; +} + +/** Extracted article data structure. */ +export interface ArticleData { + /** Best resolved URL of the article. */ + url?: string; + /** Alternative URLs discovered (canonical, shortlink, amphtml). */ + links?: string[]; + /** Article title. */ + title?: string; + /** Short description or excerpt. */ + description?: string; + /** Main image URL. */ + image?: string; + /** Site favicon URL. */ + favicon?: string; + /** Author name. */ + author?: string; + /** Extracted article HTML content. */ + content?: string; + /** Original publisher/source domain. */ + source?: string; + /** Publication date string. */ + published?: string; + /** Estimated time to read in seconds. */ + ttr?: number; + /** Page type (e.g. "article"). */ + type?: string; +} + +/** + * Load and extract article data from a URL or HTML string. + * + * @param input - URL or HTML string to extract from + * @param parserOptions - Options for parsing + * @param fetcher - Custom fetch function. Defaults to globalThis.fetch. + * @returns Extracted article data or null + */ +export const extract = async ( + input: string, + parserOptions: ParserOptions = {}, + fetcher: Fetcher = globalThis.fetch, +): Promise => { + if (!isString(input)) { + throw new Error("Input must be a string"); + } + + if (!isValidUrl(input)) { + return parseFromHtml(input, "", parserOptions); + } + const buffer = await retrieve(input, fetcher); + const text = buffer ? new TextDecoder().decode(buffer).trim() : ""; + if (!text) { + return null; + } + const charset = getCharset(text); + const decoder = new TextDecoder(charset); + const html = decoder.decode(buffer); + return parseFromHtml(html, input, parserOptions); +}; + +/** + * Extract article data from an HTML string directly. + * + * @param html - Raw HTML content + * @param url - Source URL for resolving relative links + * @param parserOptions - Options for parsing + * @returns Extracted article data or null + */ +// deno-lint-ignore require-await +export const extractFromHtml = async ( + html: string, + url?: string, + parserOptions: ParserOptions = {}, +): Promise => { + return parseFromHtml(html, url || "", parserOptions); +}; + +export { + addTransformations, + removeTransformations, +} from "./utils/transformation.ts"; diff --git a/src/utils/extractLdSchema.js b/src/utils/extractLdSchema.js deleted file mode 100644 index 2f9584b5..00000000 --- a/src/utils/extractLdSchema.js +++ /dev/null @@ -1,107 +0,0 @@ -// utils -> extractLdSchema.js - -import { isArray, isObject, isString } from '@pwshub/bellajs' - -/** - * Allowed JSON-LD schema types that indicate an article or webpage. - * - * @type {string[]} - */ -const typeSchemas = [ - 'aboutpage', - 'checkoutpage', - 'collectionpage', - 'contactpage', - 'faqpage', - 'itempage', - 'medicalwebpage', - 'profilepage', - 'qapage', - 'realestatelisting', - 'searchresultspage', - 'webpage', - 'website', - 'article', - 'advertisercontentarticle', - 'newsarticle', - 'analysisnewsarticle', - 'askpublicnewsarticle', - 'backgroundnewsarticle', - 'opinionnewsarticle', - 'reportagenewsarticle', - 'reviewnewsarticle', - 'report', - 'satiricalarticle', - 'scholarlyarticle', - 'medicalscholarlyarticle', -] - -/** - * Mapping from entry keys to JSON-LD attribute names. - * - * @type {Object} - */ -const attributeLists = { - description: 'description', - image: 'image', - author: 'author', - published: 'datePublished', - type: '@type', -} - -/** - * Safely parse a JSON string, returning an empty object on failure. - * - * @param {string} text - JSON string to parse - * @returns {Object} Parsed object or empty object - */ -const parseJson = (text) => { - try { - return JSON.parse(text) - } catch { - return {} - } -} - -/** - * Check if the given JSON-LD object has an allowed schema type. - * - * @param {Object} ldJson - Parsed JSON-LD object - * @returns {boolean} True if type is in the allowed list - */ -const isAllowedLdJsonType = (ldJson) => { - const rootLdJsonType = ldJson['@type'] || '' - const arr = isArray(rootLdJsonType) ? rootLdJsonType : [rootLdJsonType] - const ldJsonTypes = arr.filter(x => !!x) - return ldJsonTypes.length > 0 && ldJsonTypes.some(x => typeSchemas.includes(x.toLowerCase())) -} - -/** - * Parses JSON-LD data from a document and populates an entry object. - * Only populates if the original entry object is empty or undefined. - * - * @param {Document} document - The HTML Document - * @param {Object} entry - The entry object to merge/populate with JSON-LD. - * @returns {Object} The entry object after being merged/populated with data. - */ -export default (document, entry) => { - const ldSchemas = document.querySelectorAll('script[type="application/ld+json"]') - ldSchemas.forEach(ldSchema => { - const ldJson = parseJson(ldSchema.textContent.replace(/[\n\r\t]/g, '')) - if (ldJson && isAllowedLdJsonType(ldJson)) { - for (const [key, attr] of Object.entries(attributeLists)) { - if (!entry[key] || !ldJson[attr]) { - continue - } - - const keyValue = ldJson[attr] - const val = isArray(keyValue) ? keyValue[0] : isObject(keyValue) ? keyValue?.name || '' : keyValue - if (isString(val) && val !== '') { - entry[key] = val.trim() - } - } - } - }) - - return entry -} diff --git a/src/utils/extractLdSchema.ts b/src/utils/extractLdSchema.ts new file mode 100644 index 00000000..179b4f95 --- /dev/null +++ b/src/utils/extractLdSchema.ts @@ -0,0 +1,89 @@ +// utils -> extractLdSchema.ts + +import { isArray, isObject, isString } from "@pwshub/bellajs"; + +const typeSchemas = [ + "aboutpage", + "checkoutpage", + "collectionpage", + "contactpage", + "faqpage", + "itempage", + "medicalwebpage", + "profilepage", + "qapage", + "realestatelisting", + "searchresultspage", + "webpage", + "website", + "article", + "advertisercontentarticle", + "newsarticle", + "analysisnewsarticle", + "askpublicnewsarticle", + "backgroundnewsarticle", + "opinionnewsarticle", + "reportagenewsarticle", + "reviewnewsarticle", + "report", + "satiricalarticle", + "scholarlyarticle", + "medicalscholarlyarticle", +]; + +const attributeLists: Record = { + description: "description", + image: "image", + author: "author", + published: "datePublished", + type: "@type", +}; + +const parseJson = (text: string): Record => { + try { + return JSON.parse(text); + } catch { + return {}; + } +}; + +const isAllowedLdJsonType = (ldJson: Record): boolean => { + const rootLdJsonType = ldJson["@type"] || ""; + const arr = isArray(rootLdJsonType) + ? rootLdJsonType as string[] + : [rootLdJsonType as string]; + const ldJsonTypes = arr.filter((x) => !!x); + return ldJsonTypes.length > 0 && + ldJsonTypes.some((x) => typeSchemas.includes(x.toLowerCase())); +}; + +export default ( + document: Document, + entry: Record, +): Record => { + const ldSchemas = document.querySelectorAll( + 'script[type="application/ld+json"]', + ); + ldSchemas.forEach((ldSchema: Element) => { + const ldJson = parseJson(ldSchema.textContent.replace(/[\n\r\t]/g, "")); + if (ldJson && isAllowedLdJsonType(ldJson)) { + for (const [key, attr] of Object.entries(attributeLists)) { + if (entry[key] || !ldJson[attr]) { + continue; + } + + const keyValue = ldJson[attr]; + const val = isArray(keyValue) + ? (keyValue as unknown[])[0] + : isObject(keyValue) + ? (keyValue as Record)?.name || "" + : keyValue; + if (isString(val) && val !== "") { + entry[key] = (val as string).trim(); + } + } + } + }); + + return entry; +}; diff --git a/src/utils/extractMetaData.js b/src/utils/extractMetaData.js deleted file mode 100644 index 315e7fe7..00000000 --- a/src/utils/extractMetaData.js +++ /dev/null @@ -1,154 +0,0 @@ -// utils -> extractMetaData - -import { DOMParser } from 'linkedom' -import extractLdSchema from './extractLdSchema.js' -import findDate from './findDate.js' - -/** - * @param {Element} node - * @param {Object} attributeLists - * @returns {?{key: string, content: string}} - */ -function getMetaContentByNameOrProperty (node, attributeLists) { - const content = node.getAttribute('content') - if (!content) return null - - const property = node - .getAttribute('property')?.toLowerCase() ?? - node.getAttribute('itemprop')?.toLowerCase() - - const name = node.getAttribute('name')?.toLowerCase() - - for (const [key, attrs] of Object.entries(attributeLists)) { - if (attrs.includes(property) || attrs.includes(name)) { - return { key, content } - } - } - - return null -} - -/** - * @param html {string} - * @returns {{image: string, author: string, amphtml: string, description: string, canonical: string, source: string, published: string, title: string, url: string, shortlink: string, favicon: string, type: string}} - */ -export default (html) => { - const entry = { - url: '', - shortlink: '', - amphtml: '', - canonical: '', - title: '', - description: '', - image: '', - author: '', - source: '', - published: '', - favicon: '', - type: '', - } - - const sourceAttrs = [ - 'application-name', - 'og:site_name', - 'twitter:site', - 'dc.title', - ] - const urlAttrs = [ - 'og:url', - 'twitter:url', - 'parsely-link', - ] - const titleAttrs = [ - 'title', - 'og:title', - 'twitter:title', - 'parsely-title', - ] - const descriptionAttrs = [ - 'description', - 'og:description', - 'twitter:description', - 'parsely-description', - ] - const imageAttrs = [ - 'image', - 'og:image', - 'og:image:url', - 'og:image:secure_url', - 'twitter:image', - 'twitter:image:src', - 'parsely-image-url', - ] - const authorAttrs = [ - 'author', - 'creator', - 'og:creator', - 'article:author', - 'twitter:creator', - 'dc.creator', - 'parsely-author', - ] - const publishedTimeAttrs = [ - 'article:published_time', - 'article:modified_time', - 'og:updated_time', - 'dc.date', - 'dc.date.issued', - 'dc.date.created', - 'dc:created', - 'dcterms.date', - 'datepublished', - 'datemodified', - 'updated_time', - 'modified_time', - 'published_time', - 'release_date', - 'date', - 'parsely-pub-date', - ] - const typeAttrs = [ - 'og:type', - ] - - const attributeLists = { - source: sourceAttrs, - url: urlAttrs, - title: titleAttrs, - description: descriptionAttrs, - image: imageAttrs, - author: authorAttrs, - published: publishedTimeAttrs, - type: typeAttrs, - } - - const doc = new DOMParser().parseFromString(html, 'text/html') - entry.title = doc.querySelector('head > title')?.innerText - - Array.from(doc.getElementsByTagName('link')).forEach(node => { - const rel = node.getAttribute('rel') - const href = node.getAttribute('href') - if (rel && href) { - entry[rel] = href - if (rel === 'icon' || rel === 'shortcut icon') { - entry.favicon = href - } - } - }) - - Array.from(doc.getElementsByTagName('meta')).forEach(node => { - const result = getMetaContentByNameOrProperty(node, attributeLists) - const val = result?.content || '' - if (val !== '') { - entry[result.key] = val - } - }) - - const metadata = extractLdSchema(doc, entry) - - if (!metadata.published) { - metadata.published = findDate(doc) || '' - } - - return metadata -} diff --git a/src/utils/extractMetaData.test.js b/src/utils/extractMetaData.test.js deleted file mode 100644 index 00fec7c5..00000000 --- a/src/utils/extractMetaData.test.js +++ /dev/null @@ -1,57 +0,0 @@ -// extractMetaData.test -import { describe, it } from 'node:test' -import assert from 'node:assert' - -import { readFileSync } from 'node:fs' - -import { isObject, hasProperty } from '@pwshub/bellajs' - -import extractMetaData from './extractMetaData.js' - -const keys = 'url shortlink amphtml canonical title description image author source published favicon type'.split(' ') - -function isDateString (date) { - if (typeof date !== 'string') return false - const d = new Date(date) - return !isNaN(d.getTime()) -} - -describe('test extractMetaData', () => { - it('test extractMetaData(good content)', async () => { - const html = readFileSync('./test-data/regular-article.html', 'utf8') - const result = extractMetaData(html) - assert.ok(isObject(result)) - keys.forEach((k) => { - assert.ok(hasProperty(result, k)) - }) - }) - - it('test extractMetaData(json ld schema content)', async () => { - const html = readFileSync('./test-data/regular-article-json-ld.html', 'utf8') - const result = extractMetaData(html) - assert.ok(isObject(result)) - keys.forEach((k) => { - assert.ok(hasProperty(result, k)) - }) - }) - - it('test extractMetaData(find date)', async () => { - const html1 = readFileSync('./test-data/regular-article-date-time.html', 'utf8') - const html2 = readFileSync('./test-data/regular-article-date-itemprop.html', 'utf8') - const html3 = readFileSync('./test-data/regular-article-date-span.html', 'utf8') - const result1 = extractMetaData(html1) - const result2 = extractMetaData(html2) - const result3 = extractMetaData(html3) - assert.ok(isObject(result1)) - assert.ok(isObject(result2)) - assert.ok(isObject(result3)) - keys.forEach((k) => { - assert.ok(hasProperty(result1, k)) - assert.ok(hasProperty(result3, k)) - assert.ok(hasProperty(result3, k)) - }) - assert.ok(isDateString(result1.published)) - assert.ok(isDateString(result2.published)) - assert.ok(isDateString(result3.published)) - }) -}) diff --git a/src/utils/extractMetaData.ts b/src/utils/extractMetaData.ts new file mode 100644 index 00000000..8178c3ca --- /dev/null +++ b/src/utils/extractMetaData.ts @@ -0,0 +1,148 @@ +// utils -> extractMetaData.ts + +import { DOMParser } from "linkedom"; +import extractLdSchema from "./extractLdSchema.ts"; +import findDate from "./findDate.ts"; + +function getMetaContentByNameOrProperty( + node: Element, + attributeLists: Record, +): { key: string; content: string } | null { + const content = node.getAttribute("content"); + if (!content) return null; + + const property = node + .getAttribute("property")?.toLowerCase() ?? + node.getAttribute("itemprop")?.toLowerCase(); + + const name = node.getAttribute("name")?.toLowerCase(); + + for (const [key, attrs] of Object.entries(attributeLists)) { + if (attrs.includes(property || "") || attrs.includes(name || "")) { + return { key, content }; + } + } + + return null; +} + +export default (html: string): Record => { + const entry: Record = { + url: "", + shortlink: "", + amphtml: "", + canonical: "", + title: "", + description: "", + image: "", + author: "", + source: "", + published: "", + favicon: "", + type: "", + }; + + const sourceAttrs = [ + "application-name", + "og:site_name", + "twitter:site", + "dc.title", + ]; + const urlAttrs = [ + "og:url", + "twitter:url", + "parsely-link", + ]; + const titleAttrs = [ + "title", + "og:title", + "twitter:title", + "parsely-title", + ]; + const descriptionAttrs = [ + "description", + "og:description", + "twitter:description", + "parsely-description", + ]; + const imageAttrs = [ + "image", + "og:image", + "og:image:url", + "og:image:secure_url", + "twitter:image", + "twitter:image:src", + "parsely-image-url", + ]; + const authorAttrs = [ + "author", + "creator", + "og:creator", + "article:author", + "twitter:creator", + "dc.creator", + "parsely-author", + ]; + const publishedTimeAttrs = [ + "article:published_time", + "article:modified_time", + "og:updated_time", + "dc.date", + "dc.date.issued", + "dc.date.created", + "dc:created", + "dcterms.date", + "datepublished", + "datemodified", + "updated_time", + "modified_time", + "published_time", + "release_date", + "date", + "parsely-pub-date", + ]; + const typeAttrs = [ + "og:type", + ]; + + const attributeLists: Record = { + source: sourceAttrs, + url: urlAttrs, + title: titleAttrs, + description: descriptionAttrs, + image: imageAttrs, + author: authorAttrs, + published: publishedTimeAttrs, + type: typeAttrs, + }; + + const doc = new DOMParser().parseFromString(html, "text/html"); + entry.title = doc.querySelector("head > title")?.innerText || ""; + + Array.from(doc.getElementsByTagName("link")).forEach((node) => { + const rel = node.getAttribute("rel"); + const href = node.getAttribute("href"); + if (rel && href) { + entry[rel] = href; + if (rel === "icon" || rel === "shortcut icon") { + entry.favicon = href; + } + } + }); + + Array.from(doc.getElementsByTagName("meta")).forEach((node) => { + const result = getMetaContentByNameOrProperty(node, attributeLists); + const val = result?.content || ""; + if (val !== "" && result) { + entry[result.key] = val; + } + }); + + const metadata = extractLdSchema(doc as any, entry); + + if (!metadata.published) { + metadata.published = findDate(doc as any) || ""; + } + + return metadata; +}; diff --git a/src/utils/extractWithReadability.js b/src/utils/extractWithReadability.js deleted file mode 100644 index 56833d9b..00000000 --- a/src/utils/extractWithReadability.js +++ /dev/null @@ -1,42 +0,0 @@ -// utils/extractWithReadability - -import { Readability } from '@mozilla/readability' -import { DOMParser } from 'linkedom' -import { isString } from '@pwshub/bellajs' - -/** - * Extract main article content from HTML using Mozilla Readability. - * - * @param {string} html - Raw HTML content - * @param {string} [url=''] - Source URL for resolving relative paths - * @returns {string|null} Extracted article HTML or null - */ -export default (html, url = '') => { - if (!isString(html)) { - return null - } - const doc = new DOMParser().parseFromString(html, 'text/html') - const base = doc.createElement('base') - base.setAttribute('href', url) - doc.head.appendChild(base) - const reader = new Readability(doc, { - keepClasses: true, - }) - const result = reader.parse() ?? {} - return result.textContent ? result.content : null -} - -/** - * Extract article title from HTML using Mozilla Readability. - * - * @param {string} html - Raw HTML content - * @returns {string|null} Extracted title or null - */ -export function extractTitleWithReadability (html) { - if (!isString(html)) { - return null - } - const doc = new DOMParser().parseFromString(html, 'text/html') - const reader = new Readability(doc) - return reader._getArticleTitle() || null -} diff --git a/src/utils/extractWithReadability.test.js b/src/utils/extractWithReadability.test.js deleted file mode 100644 index 80f4bdec..00000000 --- a/src/utils/extractWithReadability.test.js +++ /dev/null @@ -1,43 +0,0 @@ -// extractWithReadability.test - -import { describe, it } from 'node:test' -import assert from 'node:assert' - -import { readFileSync } from 'node:fs' - -import { isString } from '@pwshub/bellajs' - -import extractWithReadability, { extractTitleWithReadability } from './extractWithReadability.js' - -describe('test extractWithReadability()', () => { - it('extract from good html content', async () => { - const html = readFileSync('./test-data/regular-article.html', 'utf8') - const result = extractWithReadability(html, 'https://foo.bar') - assert.ok(isString(result)) - assert.ok(result.length > 200) - assert.ok(result.includes('')) - }) - - it('extract from bad html content', async () => { - assert.equal(extractWithReadability(null), null) - assert.equal(extractWithReadability({}), null) - assert.equal(extractWithReadability('
'), null) - }) - - it('extract title only', async () => { - const html = readFileSync('./test-data/regular-article.html', 'utf8') - const result = extractTitleWithReadability(html) - assert.equal(result, 'Article title here - ArticleParser') - }) - - it('extract title from page without title', async () => { - const html = readFileSync('./test-data/html-no-title.html', 'utf8') - const result = extractTitleWithReadability(html) - assert.equal(result, null) - }) - - it('extract title from non-string', async () => { - const result = extractTitleWithReadability({}) - assert.equal(result, null) - }) -}) diff --git a/src/utils/extractWithReadability.ts b/src/utils/extractWithReadability.ts new file mode 100644 index 00000000..e206784a --- /dev/null +++ b/src/utils/extractWithReadability.ts @@ -0,0 +1,29 @@ +// utils -> extractWithReadability.ts + +import { Readability } from "@mozilla/readability"; +import { DOMParser } from "linkedom"; +import { isString } from "@pwshub/bellajs"; + +export default (html: string, url = ""): string | null => { + if (!isString(html)) { + return null; + } + const doc = new DOMParser().parseFromString(html, "text/html"); + const base = doc.createElement("base", {}); + base.setAttribute("href", url); + doc.head.appendChild(base); + const reader = new Readability(doc as unknown as Document, { + keepClasses: true, + }); + const result: Record = reader.parse() ?? {}; + return result.textContent ? result.content as string : null; +}; + +export function extractTitleWithReadability(html: string): string | null { + if (!isString(html)) { + return null; + } + const doc = new DOMParser().parseFromString(html, "text/html"); + const reader = new Readability(doc as unknown as Document); + return (reader as any)._getArticleTitle() || null; +} diff --git a/src/utils/findDate.js b/src/utils/findDate.js deleted file mode 100644 index 242eaa68..00000000 --- a/src/utils/findDate.js +++ /dev/null @@ -1,59 +0,0 @@ - -/** - * Convert date format to YYYY-MM-DD - * - * @param {string} dateString - * @returns {string} YYYY-MM-DD - */ -function convertDateFormat (dateString) { - const parts = dateString.split('/') - if (parts.length !== 3) return dateString - - let year, month, day - - if (parts[0].length === 4 || parseInt(parts[0]) > 31) { - [year, month, day] = parts - } else if (parseInt(parts[0]) > 12) { - [day, month, year] = parts - } else { - [month, day, year] = parts - } - - year = year.length === 2 ? '20' + year : year - return `${year}-${month.padStart(2, '0')}-${day.padStart(2, '0')}T00:00:00` -} - -/** - * Look for the publication date in the body of the content. - * - * @param {Document} document - The HTML Document - * @returns {string} The date string - */ -export default function (doc) { - const datePatterns = [ - /\d{4}-\d{2}-\d{2}/, - /\d{1,2}\/\d{1,2}\/\d{2,4}/, - ] - - const findDate = (element) => { - for (const pattern of datePatterns) { - const match = element.textContent.match(pattern) - if (match) return convertDateFormat(match[0]) - } - return '' - } - - const priorityElements = doc.querySelectorAll('time, [datetime], [itemprop~=datePublished], [itemprop~=dateCreated]') - for (const el of priorityElements) { - const date = el.getAttribute('datetime') || el.getAttribute('content') || findDate(el) - if (date) return date - } - - const secondaryElements = doc.querySelectorAll('p, span, div') - for (const el of secondaryElements) { - const date = findDate(el) - if (date) return date - } - - return '' -} diff --git a/src/utils/findDate.ts b/src/utils/findDate.ts new file mode 100644 index 00000000..b5b361cc --- /dev/null +++ b/src/utils/findDate.ts @@ -0,0 +1,51 @@ +// utils -> findDate.ts + +function convertDateFormat(dateString: string): string { + const parts = dateString.split("/"); + if (parts.length !== 3) return dateString; + + let year: string, month: string, day: string; + + if (parts[0].length === 4 || parseInt(parts[0]) > 31) { + [year, month, day] = parts; + } else if (parseInt(parts[0]) > 12) { + [day, month, year] = parts; + } else { + [month, day, year] = parts; + } + + year = year.length === 2 ? "20" + year : year; + return `${year}-${month.padStart(2, "0")}-${day.padStart(2, "0")}T00:00:00`; +} + +export default function findDate(doc: Document): string { + const datePatterns = [ + /\d{4}-\d{2}-\d{2}/, + /\d{1,2}\/\d{1,2}\/\d{2,4}/, + ]; + + const findDateInElement = (element: Element): string => { + for (const pattern of datePatterns) { + const match = element.textContent?.match(pattern); + if (match) return convertDateFormat(match[0]); + } + return ""; + }; + + const priorityElements = doc.querySelectorAll( + "time, [datetime], [itemprop~=datePublished], [itemprop~=dateCreated]", + ); + for (const el of Array.from(priorityElements)) { + const date = el.getAttribute("datetime") || el.getAttribute("content") || + findDateInElement(el); + if (date) return date; + } + + const secondaryElements = doc.querySelectorAll("p, span, div"); + for (const el of Array.from(secondaryElements)) { + const date = findDateInElement(el); + if (date) return date; + } + + return ""; +} diff --git a/src/utils/html.js b/src/utils/html.js deleted file mode 100644 index 33bbaa6f..00000000 --- a/src/utils/html.js +++ /dev/null @@ -1,100 +0,0 @@ -// utils -> html - -import { DOMParser } from 'linkedom' -import sanitize from 'sanitize-html' -import { pipe } from '@pwshub/bellajs' - -import { getSanitizeHtmlOptions } from '../config.js' - -/** - * Lightweight HTML sanitization that fixes structural issues - * without stripping any tags or attributes. - * - * @param {string} html - Raw HTML input - * @returns {string} Sanitized HTML (all tags/attributes preserved) - */ -export const purify = (html) => { - return sanitize(html, { - allowedTags: false, - allowedAttributes: false, - allowVulnerableTags: true, - }) -} - -/** - * Regex matching strings that consist entirely of whitespace characters. - * - * @type {RegExp} - */ -const WS_REGEXP = /^[\s\f\n\r\t\u1680\u180e\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000\ufeff\x09\x0a\x0b\x0c\x0d\x20\xa0]+$/ // eslint-disable-line - -/** - * Collapse multiple consecutive line breaks into single newlines, - * and remove lines that are entirely whitespace. - * - * @param {string} str - Input string - * @returns {string} Cleaned string - */ -const stripMultiLinebreaks = (str) => { - return str.replace(/(\r\n|\n|\u2424){2,}/g, '\n').split('\n').map((line) => { - return WS_REGEXP.test(line) ? line.trim() : line - }).filter((line) => { - return line.length > 0 - }).join('\n') -} - -/** - * Replace all-whitespace sequences with a single space. - * - * @param {string} str - Input string - * @returns {string} Cleaned string - */ -const stripMultispaces = (str) => { - return str.replace(WS_REGEXP, ' ').trim() -} - -/** - * Detect HTML character encoding from meta tags. - * - * @param {string} html - Raw HTML content - * @returns {string} Charset name (defaults to 'utf8') - */ -export const getCharset = (html) => { - const doc = new DOMParser().parseFromString(html, 'text/html') - const m = doc.querySelector('meta[charset]') || null - let charset = m ? m.getAttribute('charset') : '' - if (!charset) { - const h = doc.querySelector('meta[http-equiv="content-type"]') || null - charset = h ? h.getAttribute('content')?.split(';')[1]?.replace('charset=', '')?.trim() : '' - } - return charset?.toLowerCase() || 'utf8' -} - -/** - * Final cleanup of extracted article content: - * sanitize to allowed tags, collapse whitespace. - * - * @param {string} inputHtml - Extracted article HTML - * @returns {string} Cleaned HTML string - */ -export const cleanify = (inputHtml) => { - const doc = new DOMParser().parseFromString(inputHtml, 'text/html') - const html = doc.documentElement.innerHTML - return pipe( - input => sanitize(input, getSanitizeHtmlOptions()), - input => stripMultiLinebreaks(input), - input => stripMultispaces(input) - )(html) -} - -/** - * Count the number of img tags in HTML content. - * - * @param {string} html - HTML content - * @returns {number} Number of img elements - */ -export const countImages = (html) => { - const doc = new DOMParser().parseFromString(html, 'text/html') - const imgTags = doc.querySelectorAll('img') || [] - return imgTags.length -} diff --git a/src/utils/html.test.js b/src/utils/html.test.js deleted file mode 100644 index f60eb71b..00000000 --- a/src/utils/html.test.js +++ /dev/null @@ -1,23 +0,0 @@ -// html.test -import { describe, it } from 'node:test' -import assert from 'node:assert' - -import { readFileSync } from 'node:fs' - -import { isString } from '@pwshub/bellajs' - -import { - cleanify -} from './html.js' - -describe('test cleanify() method', () => { - it('check if unwanted elements/attributes removed', () => { - const html = readFileSync('./test-data/regular-article.html', 'utf8') - assert.ok(html.includes('
4746 Kelly Drive, West Virginia
')) - assert.ok(html.includes('')) - const result = cleanify(html) - assert.ok(isString(result)) - assert.equal(result.includes('
4746 Kelly Drive, West Virginia
'), false) - assert.equal(result.includes(''), false) - }) -}) diff --git a/src/utils/html.ts b/src/utils/html.ts new file mode 100644 index 00000000..beb5a4ed --- /dev/null +++ b/src/utils/html.ts @@ -0,0 +1,102 @@ +// utils -> html.ts + +import { DOMParser } from "linkedom"; + +export interface SanitizeOptions { + allowedTags: string[]; + allowedAttributes: Record; + allowedIframeDomains: string[]; +} + +const WS_REGEXP = + /^[\s\f\n\r\t\u1680\u180e\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000\ufeff\x09\x0a\x0b\x0c\x0d\x20\xa0]+$/; + +const stripMultiLinebreaks = (str: string): string => { + return str.replace(/(\r\n|\n|\u2424){2,}/g, "\n").split("\n").map((line) => { + return WS_REGEXP.test(line) ? line.trim() : line; + }).filter((line) => { + return line.length > 0; + }).join("\n"); +}; + +const stripMultispaces = (str: string): string => { + return str.replace(WS_REGEXP, " ").trim(); +}; + +export const getCharset = (html: string): string => { + const doc = new DOMParser().parseFromString(html, "text/html"); + const m = doc.querySelector("meta[charset]") || null; + let charset = m ? m.getAttribute("charset") : ""; + if (!charset) { + const h = doc.querySelector('meta[http-equiv="content-type"]') || null; + charset = h + ? h.getAttribute("content")?.split(";")[1]?.replace("charset=", "") + ?.trim() + : ""; + } + return charset?.toLowerCase() || "utf8"; +}; + +export const cleanify = ( + inputHtml: string, + options: SanitizeOptions, +): string => { + const { allowedTags, allowedAttributes, allowedIframeDomains } = options; + const tagSet = new Set(allowedTags.map((t) => t.toLowerCase())); + const doc = new DOMParser().parseFromString(inputHtml, "text/html"); + + const sanitize = (node: ChildNode) => { + if (node.nodeType !== 1) return; + const el = node as Element; + const tag = el.tagName.toLowerCase(); + + if (!tagSet.has(tag)) { + el.parentNode?.removeChild(el); + return; + } + + if (tag === "iframe") { + const src = el.getAttribute("src") || ""; + try { + const hostname = new URL(src).hostname.replace(/^www\./, ""); + const allowed = allowedIframeDomains.some((d) => + hostname === d || hostname.endsWith("." + d) + ); + if (!allowed) { + el.parentNode?.removeChild(el); + return; + } + } catch { + el.parentNode?.removeChild(el); + return; + } + } + + const allowed = allowedAttributes[tag]; + const allowedSet = allowed ? new Set(allowed) : null; + for (let i = el.attributes.length - 1; i >= 0; i--) { + const attr = el.attributes[i]; + if (!allowedSet || !allowedSet.has(attr.name)) { + el.removeAttribute(attr.name); + } + } + + const children = Array.from(node.childNodes); + for (const child of children) { + sanitize(child); + } + }; + + const children = Array.from(doc.documentElement.childNodes); + for (const child of children) { + sanitize(child); + } + + return stripMultiLinebreaks(stripMultispaces(doc.documentElement.innerHTML)); +}; + +export const countImages = (html: string): number => { + const doc = new DOMParser().parseFromString(html, "text/html"); + const imgTags = doc.querySelectorAll("img") || []; + return imgTags.length; +}; diff --git a/src/utils/linker.js b/src/utils/linker.js deleted file mode 100644 index 59ac04fa..00000000 --- a/src/utils/linker.js +++ /dev/null @@ -1,185 +0,0 @@ -// utils -> linker - -import { DOMParser } from 'linkedom' - -import { findBestMatch } from './similarity.js' - -/** - * Check if a string is a valid HTTP or HTTPS URL. - * - * @param {string} [url=''] - URL string to validate - * @returns {boolean} True if valid HTTP(S) URL - */ -export const isValid = (url = '') => { - try { - const ourl = new URL(url) - return ourl !== null && ourl.protocol.startsWith('http') - } catch { - return false - } -} - -/** - * Pick the URL that best matches the article title using string similarity. - * - * @param {string[]} [candidates=[]] - Candidate URLs - * @param {string} [title=''] - Article title for comparison - * @returns {string} Best matching URL - */ -export const chooseBestUrl = (candidates = [], title = '') => { - const ranking = findBestMatch(title, candidates) - return ranking.bestMatch.target -} - -/** - * Resolve a relative URL against a base URL. - * - * @param {string} [fullUrl=''] - Base URL - * @param {string} [relativeUrl=''] - Relative URL to resolve - * @returns {string} Absolute URL or empty string on failure - */ -export const absolutify = (fullUrl = '', relativeUrl = '') => { - try { - const result = new URL(relativeUrl, fullUrl) - return result.toString() - } catch { - return '' - } -} - -/** - * Tracking and analytics query parameters to strip from URLs. - * - * @type {string[]} - */ -const blacklistKeys = [ - 'CNDID', - '__twitter_impression', - '_hsenc', - '_openstat', - 'action_object_map', - 'action_ref_map', - 'action_type_map', - 'amp', - 'fb_action_ids', - 'fb_action_types', - 'fb_ref', - 'fb_source', - 'fbclid', - 'ga_campaign', - 'ga_content', - 'ga_medium', - 'ga_place', - 'ga_source', - 'ga_term', - 'gs_l', - 'hmb_campaign', - 'hmb_medium', - 'hmb_source', - 'mbid', - 'mc_cid', - 'mc_eid', - 'mkt_tok', - 'referrer', - 'spJobID', - 'spMailingID', - 'spReportId', - 'spUserID', - 'utm_brand', - 'utm_campaign', - 'utm_cid', - 'utm_content', - 'utm_int', - 'utm_mailing', - 'utm_medium', - 'utm_name', - 'utm_place', - 'utm_pubreferrer', - 'utm_reader', - 'utm_social', - 'utm_source', - 'utm_swu', - 'utm_term', - 'utm_userid', - 'utm_viz_id', - 'wt_mc_o', - 'yclid', - 'WT.mc_id', - 'WT.mc_ev', - 'WT.srch', - 'pk_source', - 'pk_medium', - 'pk_campaign', -] - -/** - * Remove tracking parameters and hash fragment from a URL. - * - * @param {string} url - URL to clean - * @returns {string|null} Cleaned URL or null if invalid - */ -export const purify = (url) => { - try { - const pureUrl = new URL(url) - - blacklistKeys.forEach((key) => { - pureUrl.searchParams.delete(key) - }) - - return pureUrl.toString().replace(pureUrl.hash, '') - } catch { - return null - } -} - -/** - * @param inputHtml {string} - * @param url {string} - * @returns article {string} - */ -/** - * Normalize all links, images, and source elements in HTML - * by resolving relative URLs to absolute and adding target=_blank to links. - * - * @param {string} html - HTML content to normalize - * @param {string} url - Base URL for resolving relative paths - * @returns {string} Normalized HTML string - */ -export const normalize = (html, url) => { - const doc = new DOMParser().parseFromString(html, 'text/html') - - Array.from(doc.getElementsByTagName('a')).forEach((element) => { - const href = element.getAttribute('href') - if (href) { - element.setAttribute('href', absolutify(url, href)) - element.setAttribute('target', '_blank') - } - }) - - Array.from(doc.getElementsByTagName('img')).forEach((element) => { - const src = element.getAttribute('data-src') ?? element.getAttribute('src') - if (src) { - element.setAttribute('src', absolutify(url, src)) - } - }) - - Array.from(doc.getElementsByTagName('source')).forEach((element) => { - const src = element.getAttribute('src') - if (src) { - element.setAttribute('src', absolutify(url, src)) - } - }) - - return Array.from(doc.childNodes).map(element => element.outerHTML).join('') -} - -/** - * Extract the domain from a URL, stripping the www. prefix. - * - * @param {string} url - Full URL - * @returns {string} Domain name - */ -export const getDomain = (url) => { - const host = (new URL(url)).host - return host.replace('www.', '') -} diff --git a/src/utils/linker.test.js b/src/utils/linker.test.js deleted file mode 100644 index 89eac3cb..00000000 --- a/src/utils/linker.test.js +++ /dev/null @@ -1,182 +0,0 @@ -// linker.test -import { describe, it } from 'node:test' -import assert from 'node:assert' - -import { readFileSync } from 'node:fs' - -import { isString } from '@pwshub/bellajs' - -import { - chooseBestUrl, - isValid as isValidUrl, - purify as purifyUrl, - normalize as normalizeUrls, - absolutify as absolutifyUrl -} from './linker.js' - -describe('test isValidUrl()', () => { - const cases = [ - { - url: 'https://www.23hq.com', - expected: true, - }, - { - url: 'https://secure.actblue.com', - expected: true, - }, - { - url: 'https://docs.microsoft.com/en-us/azure/iot-edge/quickstart?view=iotedge-2018-06', - expected: true, - }, - { - url: 'http://192.168.1.199:8081/example/page', - expected: true, - }, - { - url: 'ftp://192.168.1.199:8081/example/page', - expected: false, - }, - { - url: '', - expected: false, - }, - { - url: null, - expected: false, - }, - { - url: { a: 'x' }, - expected: false, - }, - ] - cases.forEach(({ url, expected }) => { - it(`isValidUrl("${url}") must return "${expected}"`, () => { - const result = isValidUrl(url) - assert.equal(result, expected) - }) - }) -}) - -describe('test normalizeUrls()', () => { - it('test adding absolute URLs to all links', () => { - const bestUrl = 'https://test-url.com/burritos-for-life' - const html = readFileSync('./test-data/regular-article.html', 'utf8') - const result = normalizeUrls(html, bestUrl) - assert.ok(isString(result)) - assert.equal(result.includes('watermelon'), false) - assert.equal(result.includes('watermelon'), true) - }) - it('test adding target=_blank to all links', () => { - const bestUrl = 'https://test-url.com/burritos-for-life' - const html = readFileSync('./test-data/regular-article.html', 'utf8') - const result = normalizeUrls(html, bestUrl) - assert.ok(isString(result)) - assert.equal(result.includes('rational peach'), false) - assert.equal(result.includes('rational peach'), true) - }) -}) - -describe('test purifyUrl()', () => { - const entries = [ - { - url: '', - expected: null, - }, - { - url: {}, - expected: null, - }, - { - url: 'https://some.where/article/abc-xyz', - expected: 'https://some.where/article/abc-xyz', - }, - { - url: 'https://some.where/article/abc-xyz#name,bob', - expected: 'https://some.where/article/abc-xyz', - }, - { - url: 'https://some.where/article/abc-xyz?utm_source=news4&utm_medium=email&utm_campaign=spring-summer', - expected: 'https://some.where/article/abc-xyz', - }, - { - url: 'https://some.where/article/abc-xyz?q=3&utm_source=news4&utm_medium=email&utm_campaign=spring-summer', - expected: 'https://some.where/article/abc-xyz?q=3', - }, - { - url: 'https://some.where/article/abc-xyz?pk_source=news4&pk_medium=email&pk_campaign=spring-summer', - expected: 'https://some.where/article/abc-xyz', - }, - { - url: 'https://some.where/article/abc-xyz?q=3&pk_source=news4&pk_medium=email&pk_campaign=spring-summer', - expected: 'https://some.where/article/abc-xyz?q=3', - }, - ] - entries.forEach((entry) => { - const { - url, - expected, - } = entry - it(`purifyUrl("${url}") must become "${expected}"`, () => { - const result = purifyUrl(url) - assert.equal(result, expected) - }) - }) -}) - -describe('test absolutifyUrl()', () => { - const entries = [ - { - full: '', - expected: '', - }, - { - relative: {}, - expected: '', - }, - { - full: 'https://some.where/article/abc-xyz', - relative: 'category/page.html', - expected: 'https://some.where/article/category/page.html', - }, - { - full: 'https://some.where/article/abc-xyz', - relative: '../category/page.html', - expected: 'https://some.where/category/page.html', - }, - { - full: 'https://some.where/blog/authors/article/abc-xyz', - relative: '/category/page.html', - expected: 'https://some.where/category/page.html', - }, - { - full: 'https://some.where/article/abc-xyz', - expected: 'https://some.where/article/abc-xyz', - }, - ] - entries.forEach((entry) => { - const { - full, - relative, - expected, - } = entry - it(`absolutifyUrl("${full}", "${relative}") must become "${expected}"`, () => { - const result = absolutifyUrl(full, relative) - assert.equal(result, expected) - }) - }) -}) - -describe('test chooseBestUrl()', () => { - it('test chooseBestUrl an actual case', () => { - const title = 'Google ฤ‘รฃ ra giรก mua Fitbit' - const urls = [ - 'https://alpha.xyz/tin-tuc-kinh-doanh/-/view_content/content/2965950/google-da-ra-gia-mua-fitbit', - 'https://alpha.xyz/tin-tuc-kinh-doanh/view/2965950/907893219797', - 'https://alpha.xyz/tin-tuc-kinh-doanh/google-da-ra-gia-mua-fitbit', - 'https://a.xyz/read/google-da-ra-gia-mua-fitbit', - 'https://a.xyz/read/2965950/907893219797', - ] - const result = chooseBestUrl(urls, title) - assert.equal(result, urls[3]) - }) -}) diff --git a/src/utils/linker.ts b/src/utils/linker.ts new file mode 100644 index 00000000..c9f912b2 --- /dev/null +++ b/src/utils/linker.ts @@ -0,0 +1,140 @@ +// utils -> linker.ts + +import { DOMParser } from "linkedom"; + +import { findBestMatch } from "./similarity.ts"; + +export const isValid = (url: string = ""): boolean => { + try { + const ourl = new URL(url); + return ourl !== null && ourl.protocol.startsWith("http"); + } catch { + return false; + } +}; + +export const chooseBestUrl = ( + candidates: string[] = [], + title = "", +): string => { + const ranking = findBestMatch(title, candidates); + return ranking.bestMatch.target; +}; + +export const absolutify = (fullUrl = "", relativeUrl = ""): string => { + try { + const result = new URL(relativeUrl, fullUrl); + return result.toString(); + } catch { + return ""; + } +}; + +const blacklistKeys = [ + "CNDID", + "__twitter_impression", + "_hsenc", + "_openstat", + "action_object_map", + "action_ref_map", + "action_type_map", + "amp", + "fb_action_ids", + "fb_action_types", + "fb_ref", + "fb_source", + "fbclid", + "ga_campaign", + "ga_content", + "ga_medium", + "ga_place", + "ga_source", + "ga_term", + "gs_l", + "hmb_campaign", + "hmb_medium", + "hmb_source", + "mbid", + "mc_cid", + "mc_eid", + "mkt_tok", + "referrer", + "spJobID", + "spMailingID", + "spReportId", + "spUserID", + "utm_brand", + "utm_campaign", + "utm_cid", + "utm_content", + "utm_int", + "utm_mailing", + "utm_medium", + "utm_name", + "utm_place", + "utm_pubreferrer", + "utm_reader", + "utm_social", + "utm_source", + "utm_swu", + "utm_term", + "utm_userid", + "utm_viz_id", + "wt_mc_o", + "yclid", + "WT.mc_id", + "WT.mc_ev", + "WT.srch", + "pk_source", + "pk_medium", + "pk_campaign", +]; + +export const purify = (url: string): string | null => { + try { + const pureUrl = new URL(url); + + blacklistKeys.forEach((key) => { + pureUrl.searchParams.delete(key); + }); + + return pureUrl.toString().replace(pureUrl.hash, ""); + } catch { + return null; + } +}; + +export const normalize = (html: string, url: string): string => { + const doc = new DOMParser().parseFromString(html, "text/html"); + + Array.from(doc.getElementsByTagName("a")).forEach((element) => { + const href = element.getAttribute("href"); + if (href) { + element.setAttribute("href", absolutify(url, href)); + element.setAttribute("target", "_blank"); + } + }); + + Array.from(doc.getElementsByTagName("img")).forEach((element) => { + const src = element.getAttribute("data-src") ?? element.getAttribute("src"); + if (src) { + element.setAttribute("src", absolutify(url, src)); + } + }); + + Array.from(doc.getElementsByTagName("source")).forEach((element) => { + const src = element.getAttribute("src"); + if (src) { + element.setAttribute("src", absolutify(url, src)); + } + }); + + return Array.from(doc.childNodes).map((element) => + (element as Element).outerHTML + ).join(""); +}; + +export const getDomain = (url: string): string => { + const host = (new URL(url)).host; + return host.replace("www.", ""); +}; diff --git a/src/utils/parseFromHtml.js b/src/utils/parseFromHtml.js deleted file mode 100644 index 2df173b2..00000000 --- a/src/utils/parseFromHtml.js +++ /dev/null @@ -1,163 +0,0 @@ -// utils -> parseFromHtml - -import { - stripTags, - truncateByChar, - unique, - pipe, - getTTR -} from '@pwshub/bellajs' - -import { - purify, - cleanify, - countImages -} from './html.js' - -import { - isValid as isValidUrl, - purify as purifyUrl, - absolutify as absolutifyUrl, - normalize as normalizeUrls, - chooseBestUrl, - getDomain -} from './linker.js' - -import extractMetaData from './extractMetaData.js' - -import extractWithReadability, { - extractTitleWithReadability -} from './extractWithReadability.js' - -import { execPreParser, execPostParser } from './transformation.js' - -/** - * Build article description from meta description or text content. - * - * @param {Object} params - * @param {string} params.desc - Meta description - * @param {string} params.text - Stripped text content - * @param {number} params.threshold - Min length to use meta description - * @param {number} params.maxlen - Max chars for truncated description - * @returns {string} Final description string - */ -const summarize = ({ desc, text, threshold, maxlen }) => { - return desc.length > threshold - ? desc - : truncateByChar(text, maxlen).replace(/\n/g, ' ') -} - -/** - * Parse HTML content and extract article data. - * Orchestrates metadata extraction, URL normalization, transformations, - * Readability extraction, and content sanitization. - * - * @param {string} inputHtml - Raw HTML content - * @param {string} [inputUrl=''] - Source URL for resolving relative links - * @param {ParserOptions} [parserOptions={}] - Parsing options - * @returns {Promise} Extracted article data or null - */ -export default async (inputHtml, inputUrl = '', parserOptions = {}) => { - const pureHtml = purify(inputHtml) - const meta = extractMetaData(pureHtml) - - let title = meta.title - - const { - url, - shortlink, - amphtml, - canonical, - description: metaDesc, - image: metaImg, - author, - published, - favicon: metaFav, - type, - } = meta - - const { - wordsPerMinute = 300, - descriptionTruncateLen = 210, - descriptionLengthThreshold = 180, - contentLengthThreshold = 200, - } = parserOptions - - // gather title - if (!title) { - title = extractTitleWithReadability(pureHtml, inputUrl) - } - if (!title) { - return null - } - - // gather urls to choose the best url later - const links = unique( - [url, shortlink, amphtml, canonical, inputUrl] - .filter(isValidUrl) - .map(purifyUrl) - ) - - if (!links.length) { - return null - } - - // choose the best url, which one looks like title the most - const bestUrl = chooseBestUrl(links, title) - - const fns = pipe( - (input) => { - return normalizeUrls(input, bestUrl) - }, - (input) => { - return execPreParser(input, links) - }, - (input) => { - return extractWithReadability(input, bestUrl) - }, - (input) => { - return input ? execPostParser(input, links) : null - }, - (input) => { - return input ? cleanify(input) : null - } - ) - - const content = fns(inputHtml) - - if (!content) { - return null - } - - const textContent = stripTags(content) - if (textContent.length < contentLengthThreshold) { - return null - } - - const description = summarize({ - desc: metaDesc, - text: textContent, - threshold: descriptionLengthThreshold, - maxlen: descriptionTruncateLen, - }) - - const image = metaImg ? absolutifyUrl(bestUrl, metaImg) : '' - const favicon = metaFav ? absolutifyUrl(bestUrl, metaFav) : '' - - const imgcount = countImages(content) - - return { - url: bestUrl, - title, - description, - links, - image, - content, - author, - favicon, - source: getDomain(bestUrl), - published, - ttr: getTTR(textContent, imgcount, wordsPerMinute), - type, - } -} diff --git a/src/utils/parseFromHtml.test.js b/src/utils/parseFromHtml.test.js deleted file mode 100644 index 54512ce4..00000000 --- a/src/utils/parseFromHtml.test.js +++ /dev/null @@ -1,133 +0,0 @@ -// parseFromHtml.test -import { describe, it } from 'node:test' -import assert from 'node:assert' - -import { readFileSync } from 'node:fs' - -import { isFunction } from '@pwshub/bellajs' - -import { extractFromHtml as parseFromHtml } from '../main.js' -import { addTransformations } from './transformation.js' - -const expDesc = [ - 'Navigation here Few can name a rational peach that isn\'t a conscientious goldfish!', - 'One cannot separate snakes from plucky pomegranates?', - 'Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs.', -].join(' ') - -const cases = [ - { - input: { - desc: 'a webpage with no title', - html: readFileSync('./test-data/html-no-title.html', 'utf8'), - }, - expectation: null, - }, - { - input: { - desc: 'a webpage without link', - html: readFileSync('./test-data/html-no-link.html', 'utf8'), - }, - expectation: null, - }, - { - input: { - desc: 'a webpage with no main article', - html: readFileSync('./test-data/html-no-article.html', 'utf8'), - }, - expectation: null, - }, - { - input: { - desc: 'a webpage with a very short article', - html: readFileSync('./test-data/html-too-short-article.html', 'utf8'), - url: 'abcd', - }, - expectation: null, - }, - { - input: { - desc: 'a webpage with article but no source', - html: readFileSync('./test-data/html-article-no-source.html', 'utf8'), - }, - expectation: (result) => { - assert.equal(result.source, 'somewhere.any') - }, - }, - { - input: { - desc: 'a webpage with data-src in img tag', - html: readFileSync('./test-data/html-article-with-data-src.html', 'utf8'), - }, - expectation: (result) => { - assert.equal(result.content.includes(''), true) - assert.equal(result.content.includes(''), true) - }, - }, - { - input: { - desc: 'a webpage with regular article', - html: readFileSync('./test-data/regular-article.html', 'utf8'), - url: 'https://somewhere.com/path/to/article', - }, - expectation: (result) => { - assert.equal(result.title, 'Article title here') - assert.equal(result.description, expDesc) - assert.equal(result.content.includes(''), true) - assert.equal(result.content.includes(''), true) - }, - }, -] - -describe('test parseFromHtml', () => { - cases.forEach((acase) => { - const { input, expectation } = acase - const { desc, html, url = '' } = input - it(`check if parseFromHtml() works with ${desc}`, async () => { - const result = await parseFromHtml(html, url) - if (isFunction(expectation)) { - expectation(result) - } else { - assert.equal(result, expectation) - } - }) - }) - - it('check if parseFromHtml() works with multi transforms', async () => { - addTransformations([ - { - patterns: [ - /http(s?):\/\/need-transform.tld\/*/, - ], - post: (document) => { - document.querySelectorAll('a').forEach((node) => { - const sHtml = node.innerHTML - const link = node.getAttribute('href') - node.parentNode.replaceChild(document.createTextNode(`[link url="${link}"]${sHtml}[/link]`), node) - }) - return document - }, - }, - { - patterns: [ - /http(s?):\/\/sw.re\/*/, - ], - post: (document) => { - document.querySelectorAll('strong').forEach((node) => { - const b = document.createElement('B') - b.innerHTML = node.innerHTML - node.parentNode.replaceChild(b, node) - }) - return document - }, - }, - ]) - const html = readFileSync('./test-data/vnn-article.html', 'utf8') - const url = 'https://need-transform.tld/path/to/article' - const result = await parseFromHtml(html, url) - assert.equal(result.title, 'Article title here') - assert.equal(result.content.includes(''), false) - assert.equal(result.content.includes('[link url="https://vnn.vn/dict/watermelon"]watermelon[/link]'), true) - assert.equal(result.content.includes('in its own way'), true) - }) -}) diff --git a/src/utils/parseFromHtml.ts b/src/utils/parseFromHtml.ts new file mode 100644 index 00000000..038c0177 --- /dev/null +++ b/src/utils/parseFromHtml.ts @@ -0,0 +1,164 @@ +// utils -> parseFromHtml.ts + +import { + getTTR, + pipe, + stripTags, + truncateByChar, + unique, +} from "@pwshub/bellajs"; + +import { cleanify, countImages } from "./html.ts"; + +import { + absolutify as absolutifyUrl, + chooseBestUrl, + getDomain, + isValid as isValidUrl, + normalize as normalizeUrls, + purify as purifyUrl, +} from "./linker.ts"; + +import extractMetaData from "./extractMetaData.ts"; + +import extractWithReadability, { + extractTitleWithReadability, +} from "./extractWithReadability.ts"; + +import { execPostParser, execPreParser } from "./transformation.ts"; + +import { + defaultAllowedAttributes, + defaultAllowedIframeDomains, + defaultAllowedTags, +} from "../config.ts"; + +import type { ArticleData, ParserOptions } from "../main.ts"; + +const summarize = ( + { desc, text, threshold, maxlen }: { + desc: string; + text: string; + threshold: number; + maxlen: number; + }, +): string => { + return desc.length > threshold + ? desc + : truncateByChar(text, maxlen).replace(/\n/g, " "); +}; + +// deno-lint-ignore require-await +export default async ( + inputHtml: string, + inputUrl = "", + parserOptions: ParserOptions = {}, +): Promise => { + const meta = extractMetaData(inputHtml); + + let title = meta.title; + + const { + url, + shortlink, + amphtml, + canonical, + description: metaDesc, + image: metaImg, + author, + published, + favicon: metaFav, + type, + } = meta; + + const { + wordsPerMinute = 300, + descriptionTruncateLen = 210, + descriptionLengthThreshold = 180, + contentLengthThreshold = 200, + allowedTags = defaultAllowedTags, + allowedAttributes = defaultAllowedAttributes, + allowedIframeDomains = defaultAllowedIframeDomains, + } = parserOptions; + + if (!title) { + title = extractTitleWithReadability(inputHtml) || ""; + } + if (!title) { + return null; + } + + const links = unique( + [url, shortlink, amphtml, canonical, inputUrl] + .filter(isValidUrl) + .map(purifyUrl as (s: string) => string), + ); + + if (!links.length) { + return null; + } + + const bestUrl = chooseBestUrl(links, title); + + const fns = pipe( + (input: string) => { + return normalizeUrls(input, bestUrl); + }, + (input: string) => { + return execPreParser(input, links); + }, + (input: string) => { + return extractWithReadability(input, bestUrl); + }, + (input: string | null) => { + return input ? execPostParser(input, links) : null; + }, + (input: string | null) => { + return input + ? cleanify(input, { + allowedTags, + allowedAttributes, + allowedIframeDomains, + }) + : null; + }, + ); + + const content = fns(inputHtml); + + if (!content) { + return null; + } + + const textContent = stripTags(content); + if (textContent.length < contentLengthThreshold) { + return null; + } + + const description = summarize({ + desc: metaDesc, + text: textContent, + threshold: descriptionLengthThreshold, + maxlen: descriptionTruncateLen, + }); + + const image = metaImg ? absolutifyUrl(bestUrl, metaImg) : ""; + const favicon = metaFav ? absolutifyUrl(bestUrl, metaFav) : ""; + + const imgcount = countImages(content); + + return { + url: bestUrl, + title, + description, + links, + image, + content, + author, + favicon, + source: getDomain(bestUrl), + published, + ttr: getTTR(textContent, imgcount, wordsPerMinute), + type, + }; +}; diff --git a/src/utils/retrieve.js b/src/utils/retrieve.js deleted file mode 100644 index 06f1e596..00000000 --- a/src/utils/retrieve.js +++ /dev/null @@ -1,53 +0,0 @@ -// utils -> retrieve - -/** - * Fetch content through a proxy endpoint. - * - * @param {string} url - Target URL to fetch - * @param {Object} [options={}] - Proxy options - * @param {Object} [options.proxy={}] - Proxy configuration - * @param {string} options.proxy.target - Proxy endpoint URL - * @param {Object} [options.proxy.headers={}] - Headers for proxy request - * @param {AbortSignal} [options.signal] - Optional abort signal - * @returns {Promise} Fetch response object - */ -const profetch = async (url, options = {}) => { - const { proxy = {}, signal = null } = options - const { - target, - headers = {}, - } = proxy - const res = await fetch(target + encodeURIComponent(url), { - headers, - signal, - }) - return res -} - -/** - * Retrieve raw HTML content from a URL. - * Supports direct fetch, proxy, custom headers, agent, and abort signal. - * - * @param {string} url - URL to fetch - * @param {FetchOptions} [options={}] - Fetch configuration - * @returns {Promise} Response body as ArrayBuffer - */ -export default async (url, options = {}) => { - const { - headers = { - 'user-agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/115.0', - }, - proxy = null, - agent = null, - signal = null, - } = options - - const res = proxy ? await profetch(url, { proxy, signal }) : await fetch(url, { headers, agent, signal }) - - const status = res.status - if (status >= 400) { - throw new Error(`Request failed with error code ${status}`) - } - const buffer = await res.arrayBuffer() - return buffer -} diff --git a/src/utils/retrieve.test.js b/src/utils/retrieve.test.js deleted file mode 100644 index a410c004..00000000 --- a/src/utils/retrieve.test.js +++ /dev/null @@ -1,68 +0,0 @@ -// retrieve.test -import { describe, it } from 'node:test' -import assert from 'node:assert' - -import nock from 'nock' - -import retrieve from './retrieve.js' - -const parseUrl = (url) => { - const re = new URL(url) - return { - baseUrl: `${re.protocol}//${re.host}`, - path: re.pathname, - } -} - -describe('test retrieve() method', () => { - it('test retrieve with bad status code', async () => { - const url = 'https://some.where/bad/page' - const { baseUrl, path } = parseUrl(url) - nock(baseUrl).get(path).reply(500, 'Error 500') - assert.rejects(retrieve(url), new Error('Request failed with error code 500')) - }) - - it('test retrieve from good source', async () => { - const url = 'https://some.where/good/page' - const { baseUrl, path } = parseUrl(url) - nock(baseUrl).get(path).reply(200, '
this is content
', { - 'Content-Type': 'text/html', - }) - const buffer = await retrieve(url) - const html = Buffer.from(buffer).toString() - assert.equal(html, '
this is content
') - }) - - it('test retrieve from good source with \\r\\n', async () => { - const url = 'https://some.where/good/page' - const { baseUrl, path } = parseUrl(url) - nock(baseUrl).get(path).reply(200, '\n\r\r\n\n
this is content
\n\r\r\n\n', { - 'Content-Type': 'text/html', - }) - const buffer = await retrieve(url) - const html = Buffer.from(buffer).toString().trim() - assert.equal(html, '
this is content
') - }) - - it('test retrieve using proxy', async () => { - const url = 'https://some.where/good/source-with-proxy' - const { baseUrl, path } = parseUrl(url) - nock(baseUrl).get(path).reply(200, 'something bad', { - 'Content-Type': 'bad/thing', - }) - nock('https://proxy-server.com') - .get('/api/proxy?url=https%3A%2F%2Fsome.where%2Fgood%2Fsource-with-proxy') - .reply(200, '
this is content
', { - 'Content-Type': 'text/html', - }) - - const buffer = await retrieve(url, { - proxy: { - target: 'https://proxy-server.com/api/proxy?url=', - }, - }) - const html = Buffer.from(buffer).toString() - assert.equal(html, '
this is content
') - nock.cleanAll() - }) -}) diff --git a/src/utils/retrieve.ts b/src/utils/retrieve.ts new file mode 100644 index 00000000..6545be52 --- /dev/null +++ b/src/utils/retrieve.ts @@ -0,0 +1,15 @@ +// utils -> retrieve.ts + +export default async ( + url: string, + fetcher: (url: string) => Promise, +): Promise => { + const res = await fetcher(url); + + const status = res.status; + if (status >= 400) { + throw new Error(`Request failed with error code ${status}`); + } + const buffer = await res.arrayBuffer(); + return buffer; +}; diff --git a/src/utils/similarity.js b/src/utils/similarity.js deleted file mode 100644 index 4201ca91..00000000 --- a/src/utils/similarity.js +++ /dev/null @@ -1,48 +0,0 @@ -// similarity.js - -import { - isString, - compareTwoStrings, - isArray -} from '@pwshub/bellajs' - -/** - * Validate arguments for findBestMatch. - * - * @param {string} mainString - Reference string - * @param {string[]} targetStrings - Strings to compare against - * @returns {boolean} True if arguments are valid - */ -const areArgsValid = (mainString, targetStrings) => { - return isString(mainString) && isArray(targetStrings) - && targetStrings.length > 0 && targetStrings.every(s => isString(s)) -} - -/** - * Find the best matching string from a list using Dice coefficient. - * - * @param {string} mainString - Reference string to match against - * @param {string[]} targetStrings - Candidate strings - * @returns {{ratings: Array, bestMatch: {target: string, rating: number}, bestMatchIndex: number}} Match results with rankings - */ -export const findBestMatch = (mainString, targetStrings) => { - if (!areArgsValid(mainString, targetStrings)) { - throw new Error('Bad arguments: First argument should be a string, second should be an array of strings') - } - - const ratings = [] - let bestMatchIndex = 0 - - for (let i = 0; i < targetStrings.length; i++) { - const currentTargetString = targetStrings[i] - const currentRating = compareTwoStrings(mainString, currentTargetString) - ratings.push({ target: currentTargetString, rating: currentRating }) - if (currentRating > ratings[bestMatchIndex].rating) { - bestMatchIndex = i - } - } - - const bestMatch = ratings[bestMatchIndex] - - return { ratings: ratings, bestMatch: bestMatch, bestMatchIndex: bestMatchIndex } -} diff --git a/src/utils/similarity.ts b/src/utils/similarity.ts new file mode 100644 index 00000000..e25ff113 --- /dev/null +++ b/src/utils/similarity.ts @@ -0,0 +1,47 @@ +// utils -> similarity.ts + +import { compareTwoStrings, isArray, isString } from "@pwshub/bellajs"; + +interface MatchResult { + target: string; + rating: number; +} + +interface BestMatchResult { + ratings: MatchResult[]; + bestMatch: MatchResult; + bestMatchIndex: number; +} + +const areArgsValid = (mainString: unknown, targetStrings: unknown): boolean => { + return isString(mainString) && isArray(targetStrings) && + (targetStrings as unknown[]).length > 0 && + (targetStrings as string[]).every((s) => isString(s)); +}; + +export const findBestMatch = ( + mainString: string, + targetStrings: string[], +): BestMatchResult => { + if (!areArgsValid(mainString, targetStrings)) { + throw new Error( + "Bad arguments: First argument should be a string, second should be an array of strings", + ); + } + + const ratings: MatchResult[] = []; + let bestMatchIndex = 0; + + for (let i = 0; i < targetStrings.length; i++) { + const currentTargetString = targetStrings[i]; + const currentRating = compareTwoStrings(mainString, currentTargetString); + ratings.push({ target: currentTargetString, rating: currentRating }); + if (currentRating > ratings[bestMatchIndex].rating) { + bestMatchIndex = i; + } + } + + const bestMatch = ratings[bestMatchIndex]; + + return { ratings, bestMatch, bestMatchIndex }; +}; diff --git a/src/utils/transformation.js b/src/utils/transformation.js deleted file mode 100644 index 06aff829..00000000 --- a/src/utils/transformation.js +++ /dev/null @@ -1,122 +0,0 @@ -// utils --> transformation.js - -import { isArray, isFunction } from '@pwshub/bellajs' -import { DOMParser } from 'linkedom' - -/** - * Registered transformation rules for per-site HTML pre/post processing. - * - * @type {Transformation[]} - */ -const transformations = [] - -/** - * Add a single transformation to the registry. - * - * @param {Transformation} tn - Transformation object with patterns and handlers - * @returns {number} 1 if added, 0 if invalid - */ -const add = (tn) => { - const { patterns } = tn - if (!patterns || !isArray(patterns) || !patterns.length) { - return 0 - } - transformations.push(tn) - return 1 -} - -/** - * Register one or more transformations for per-site HTML processing. - * - * @param {Transformation|Transformation[]} tfms - Transformation(s) to add - * @returns {number} Number of transformations successfully added - */ -export const addTransformations = (tfms) => { - if (isArray(tfms)) { - return tfms.map(tfm => add(tfm)).filter(result => result === 1).length - } - return add(tfms) -} - -/** - * Remove transformations matching the given patterns. - * Calling without arguments removes all transformations. - * - * @param {RegExp[]} [patterns] - URL patterns to match for removal - * @returns {number} Number of transformations removed - */ -export const removeTransformations = (patterns) => { - if (!patterns) { - const removed = transformations.length - transformations.length = 0 - return removed - } - let removing = 0 - for (let i = transformations.length - 1; i >= 0; i--) { - const { patterns: ipatterns } = transformations[i] - const matched = ipatterns.some((ptn) => patterns.some((pattern) => String(pattern) === String(ptn))) - if (matched) { - transformations.splice(i, 1) - removing += 1 - } - } - return removing -} - -/** - * Get a copy of all registered transformations. - * - * @returns {Transformation[]} Copy of transformations array - */ -export const getTransformations = () => { - return [...transformations] -} - -/** - * Find all transformations whose patterns match any of the given URLs. - * - * @param {string|string[]} links - URL(s) to match against transformation patterns - * @returns {Transformation[]} Matching transformations - */ -export const findTransformations = (links) => { - const urls = !isArray(links) ? [links] : links - const tfms = [] - for (const transformation of transformations) { - const { patterns } = transformation - const matched = urls.some((url) => patterns.some((pattern) => pattern.test(url))) - if (matched) { - tfms.push({ - ...transformation, - }) - } - } - return tfms -} - -/** - * Run pre-extraction transformations on raw HTML. - * Mutates the DOM in place through registered pre-processor functions. - * - * @param {string} html - Raw HTML content - * @param {string[]} links - URLs to match against transformation patterns - * @returns {string} Transformed HTML string - */ -export const execPreParser = (html, links) => { - const doc = new DOMParser().parseFromString(html, 'text/html') - findTransformations(links).map(tfm => tfm.pre).filter(fn => isFunction(fn)).forEach(fn => fn(doc)) - return Array.from(doc.childNodes).map(it => it.outerHTML).join('') -} - -/** - * Run post-extraction transformations on extracted article content. - * Mutates the DOM in place through registered post-processor functions. - * - * @param {string} html - Extracted article HTML - * @param {string[]} links - URLs to match against transformation patterns - * @returns {string} Transformed HTML string - */ -export const execPostParser = (html, links) => { - const doc = new DOMParser().parseFromString(html, 'text/html') - findTransformations(links).map(tfm => tfm.post).filter(fn => isFunction(fn)).forEach(fn => fn(doc)) - return Array.from(doc.childNodes).map(it => it.outerHTML).join('') -} diff --git a/src/utils/transformation.test.js b/src/utils/transformation.test.js deleted file mode 100644 index 18a54d4a..00000000 --- a/src/utils/transformation.test.js +++ /dev/null @@ -1,194 +0,0 @@ -// transformation.test -import { describe, it as test } from 'node:test' -import assert from 'node:assert' - -import { - addTransformations, - removeTransformations, - getTransformations, - findTransformations, - execPreParser, - execPostParser -} from './transformation.js' - -describe('test transformation', () => { - test(' add one transformation object', () => { - const result = addTransformations({ - patterns: [ - /http(s?):\/\/([\w]+.)?def.tld\/*/, - ], - pre: (document) => { - return document - }, - post: (document) => { - return document - }, - }) - assert.equal(result, 1) - }) - - test(' add multi transformation object', () => { - const result = addTransformations([ - { - patterns: [ - /http(s?):\/\/google.com\/*/, - /http(s?):\/\/goo.gl\/*/, - ], - }, - { - patterns: [ - /http(s?):\/\/goo.gl\/*/, - /http(s?):\/\/google.inc\/*/, - ], - }, - ]) - assert.equal(result, 2) - }) - - test(' add transformation object without patterns', () => { - const result = addTransformations({ - pre: (document) => { - return document - }, - post: (document) => { - return document - }, - }) - assert.equal(result, 0) - }) - - test(' add transformation object without valid patterns', () => { - const result = addTransformations({ - patterns: 123, - pre: (document) => { - return document - }, - post: (document) => { - return document - }, - }) - assert.equal(result, 0) - }) - - test(' get all transformations', () => { - const result = getTransformations() - assert.equal(result.length, 3) - assert.deepEqual(result[0].patterns[0], /http(s?):\/\/([\w]+.)?def.tld\/*/) - }) - - test(' remove one transformation', () => { - addTransformations([ - { - patterns: [ - /http(s?):\/\/abc.com\/*/, - /http(s?):\/\/def.gl\/*/, - ], - }, - { - patterns: [ - /http(s?):\/\/hik.gl\/*/, - /http(s?):\/\/lmn.inc\/*/, - ], - }, - { - patterns: [ - /http(s?):\/\/opq.gl\/*/, - /http(s?):\/\/rst.inc\/*/, - ], - }, - ]) - const result = removeTransformations([ - /http(s?):\/\/goo.gl\/*/, - ]) - assert.equal(result, 2) - }) - - test(' get all transformations again', () => { - const result = getTransformations() - assert.equal(result.length, 4) - assert.deepEqual(result[3].patterns[1], /http(s?):\/\/rst.inc\/*/) - }) - - test(' find transformations', () => { - addTransformations([ - { - patterns: [ - /http(s?):\/\/def.gl\/*/, - /http(s?):\/\/uvw.inc\/*/, - ], - }, - ]) - const notFound = findTransformations([ - 'https://goo.gl/docs/article.html', - ]) - assert.deepEqual(notFound, []) - - const foundOne = findTransformations([ - 'https://lmn.inc/docs/article.html', - ]) - assert.equal(foundOne.length, 1) - - const foundTwo = findTransformations([ - 'https://def.gl/docs/article.html', - ]) - assert.equal(foundTwo.length, 2) - }) - - test(' run execPreParser', () => { - addTransformations([ - { - patterns: [ - /http(s?):\/\/xyz.com\/*/, - ], - pre: (doc) => { - doc.querySelectorAll('.adv').forEach((element) => { - element.parentNode.removeChild(element) - }) - return doc - }, - }, - ]) - const html = ` -
- hi user, this is an advertisement element -
free product now!
-
- ` - const result = execPreParser(html, 'https://xyz.com/article') - assert.equal(result.includes('hi user, this is an advertisement element'), true) - assert.equal(result.includes('
free product now!
'), false) - }) - - test(' run execPostParser', () => { - addTransformations([ - { - patterns: [ - /http(s?):\/\/xyz.com\/*/, - ], - post: (doc) => { - doc.querySelectorAll('b').forEach((element) => { - const itag = doc.createElement('i') - itag.innerHTML = element.innerHTML - element.parentNode.replaceChild(itag, element) - }) - return doc - }, - }, - ]) - const html = ` -
- hi user, -

Thank you for your feedback!

-
- ` - const result = execPostParser(html, 'https://xyz.com/article') - assert.equal(result.includes('user'), true) - assert.equal(result.includes('user'), false) - }) - - test(' remove all transformations', () => { - const result = removeTransformations() - assert.equal(result, 7) - assert.deepEqual(getTransformations(), []) - }) -}) diff --git a/src/utils/transformation.ts b/src/utils/transformation.ts new file mode 100644 index 00000000..8a1fa8d5 --- /dev/null +++ b/src/utils/transformation.ts @@ -0,0 +1,103 @@ +// utils -> transformation.ts + +import { isArray, isFunction } from "@pwshub/bellajs"; +import { DOMParser } from "linkedom"; + +export interface Transformation { + patterns: RegExp[]; + pre?: (document: Document) => Document; + post?: (document: Document) => Document; +} + +const transformations: Transformation[] = []; + +const add = (tn: Transformation): number => { + const { patterns } = tn; + if (!patterns || !isArray(patterns) || !patterns.length) { + return 0; + } + transformations.push(tn); + return 1; +}; + +/** Register one or more transformations for per-site HTML processing. */ +export const addTransformations = ( + tfms: Transformation | Transformation[], +): number => { + if (isArray(tfms)) { + return (tfms as Transformation[]).map((tfm) => add(tfm)).filter((result) => + result === 1 + ).length; + } + return add(tfms as Transformation); +}; + +/** Remove transformations matching the given patterns. Removes all if no patterns given. */ +export const removeTransformations = (patterns?: RegExp[]): number => { + if (!patterns) { + const removed = transformations.length; + transformations.length = 0; + return removed; + } + let removing = 0; + for (let i = transformations.length - 1; i >= 0; i--) { + const { patterns: ipatterns } = transformations[i]; + const matched = ipatterns.some((ptn) => + (patterns as RegExp[]).some((pattern) => String(pattern) === String(ptn)) + ); + if (matched) { + transformations.splice(i, 1); + removing += 1; + } + } + return removing; +}; + +export const getTransformations = (): Transformation[] => { + return [...transformations]; +}; + +export const findTransformations = ( + links: string | string[], +): Transformation[] => { + const urls = !isArray(links) ? [links] : links; + const tfms: Transformation[] = []; + for (const transformation of transformations) { + const { patterns } = transformation; + const matched = (urls as string[]).some((url) => + patterns.some((pattern) => pattern.test(url)) + ); + if (matched) { + tfms.push({ + ...transformation, + }); + } + } + return tfms; +}; + +export const execPreParser = ( + html: string, + links: string | string[], +): string => { + const doc = new DOMParser().parseFromString(html, "text/html"); + findTransformations(links).map((tfm) => tfm.pre).filter((fn) => + isFunction(fn) + ).forEach((fn) => (fn as (doc: Document) => Document)(doc as any)); + return Array.from(doc.childNodes).map((it) => (it as Element).outerHTML).join( + "", + ); +}; + +export const execPostParser = ( + html: string, + links: string | string[], +): string => { + const doc = new DOMParser().parseFromString(html, "text/html"); + findTransformations(links).map((tfm) => tfm.post).filter((fn) => + isFunction(fn) + ).forEach((fn) => (fn as (doc: Document) => Document)(doc as any)); + return Array.from(doc.childNodes).map((it) => (it as Element).outerHTML).join( + "", + ); +}; diff --git a/test-data/html-too-short-article.html b/test-data/html-too-short-article.html deleted file mode 100644 index 4982be3b..00000000 --- a/test-data/html-too-short-article.html +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - TechNews - - -
- Hello world, this is too short, right? -

The first fair dog is, in its own way, a lemon.

-
- - diff --git a/tests/config.test.ts b/tests/config.test.ts new file mode 100644 index 00000000..c006e2cc --- /dev/null +++ b/tests/config.test.ts @@ -0,0 +1,35 @@ +// tests/config.test.ts + +import { assertEquals } from "@std/assert"; +import { + defaultAllowedAttributes, + defaultAllowedIframeDomains, + defaultAllowedTags, +} from "../src/config.ts"; + +Deno.test("defaultAllowedTags - contains expected tags", () => { + assertEquals(defaultAllowedTags.includes("a"), true); + assertEquals(defaultAllowedTags.includes("img"), true); + assertEquals(defaultAllowedTags.includes("p"), true); + assertEquals(defaultAllowedTags.includes("script"), false); + assertEquals(defaultAllowedTags.includes("style"), false); +}); + +Deno.test("defaultAllowedAttributes - contains expected attributes", () => { + assertEquals(defaultAllowedAttributes.a, ["href", "target", "title"]); + assertEquals(defaultAllowedAttributes.img, ["src", "srcset", "alt", "title"]); + assertEquals(defaultAllowedAttributes.iframe, [ + "src", + "frameborder", + "height", + "width", + "scrolling", + "allow", + ]); +}); + +Deno.test("defaultAllowedIframeDomains - contains expected domains", () => { + assertEquals(defaultAllowedIframeDomains.includes("youtube.com"), true); + assertEquals(defaultAllowedIframeDomains.includes("vimeo.com"), true); + assertEquals(defaultAllowedIframeDomains.includes("evil.com"), false); +}); diff --git a/tests/extractMetaData.test.ts b/tests/extractMetaData.test.ts new file mode 100644 index 00000000..38a107d3 --- /dev/null +++ b/tests/extractMetaData.test.ts @@ -0,0 +1,82 @@ +// tests/extractMetaData.test.ts + +import { assertEquals } from "@std/assert"; +import { hasProperty, isObject } from "@pwshub/bellajs"; +import extractMetaData from "../src/utils/extractMetaData.ts"; + +const keys = + "url shortlink amphtml canonical title description image author source published favicon type" + .split(" "); + +function isDateString(date: unknown): boolean { + if (typeof date !== "string") return false; + const d = new Date(date); + return !isNaN(d.getTime()); +} + +Deno.test("extractMetaData - good content", () => { + const html = Deno.readTextFileSync("tests/test-data/regular-article.html"); + const result = extractMetaData(html); + assertEquals(isObject(result), true); + keys.forEach((k) => { + assertEquals(hasProperty(result, k), true); + }); +}); + +Deno.test("extractMetaData - json ld schema content", () => { + const html = Deno.readTextFileSync( + "tests/test-data/regular-article-json-ld.html", + ); + const result = extractMetaData(html); + assertEquals(isObject(result), true); + keys.forEach((k) => { + assertEquals(hasProperty(result, k), true); + }); +}); + +Deno.test("extractMetaData - json ld schema fills empty meta fields", () => { + const ldJson = { + "@context": "https://schema.org", + "@type": "NewsArticle", + author: { name: "JSON Author" }, + datePublished: "2026-01-02T03:04:05Z", + description: "JSON description", + image: "https://example.com/json.jpg", + }; + const body = "

" + "Article text. ".repeat(50) + "

"; + const html = ` +Fixture article + +

Fixture article

${body}
`; + const result = extractMetaData(html); + assertEquals(result.author, "JSON Author"); + assertEquals(result.published, "2026-01-02T03:04:05Z"); + assertEquals(result.description, "JSON description"); + assertEquals(result.image, "https://example.com/json.jpg"); +}); + +Deno.test("extractMetaData - find date", () => { + const html1 = Deno.readTextFileSync( + "tests/test-data/regular-article-date-time.html", + ); + const html2 = Deno.readTextFileSync( + "tests/test-data/regular-article-date-itemprop.html", + ); + const html3 = Deno.readTextFileSync( + "tests/test-data/regular-article-date-span.html", + ); + const result1 = extractMetaData(html1); + const result2 = extractMetaData(html2); + const result3 = extractMetaData(html3); + assertEquals(isObject(result1), true); + assertEquals(isObject(result2), true); + assertEquals(isObject(result3), true); + keys.forEach((k) => { + assertEquals(hasProperty(result1, k), true); + assertEquals(hasProperty(result2, k), true); + assertEquals(hasProperty(result3, k), true); + }); + assertEquals(isDateString(result1.published), true); + assertEquals(isDateString(result2.published), true); + assertEquals(isDateString(result3.published), true); +}); diff --git a/tests/extractWithReadability.test.ts b/tests/extractWithReadability.test.ts new file mode 100644 index 00000000..46da44d0 --- /dev/null +++ b/tests/extractWithReadability.test.ts @@ -0,0 +1,41 @@ +// tests/extractWithReadability.test.ts + +import { assertEquals } from "@std/assert"; +import { isString } from "@pwshub/bellajs"; +import extractWithReadability, { + extractTitleWithReadability, +} from "../src/utils/extractWithReadability.ts"; + +Deno.test("extractWithReadability - good html content", () => { + const html = Deno.readTextFileSync("tests/test-data/regular-article.html"); + const result = extractWithReadability(html, "https://foo.bar"); + assertEquals(isString(result), true); + assertEquals((result as string).length > 200, true); + assertEquals( + (result as string).includes(''), + true, + ); +}); + +Deno.test("extractWithReadability - bad html content", () => { + assertEquals(extractWithReadability(null as unknown as string), null); + assertEquals(extractWithReadability({} as unknown as string), null); + assertEquals(extractWithReadability("
"), null); +}); + +Deno.test("extractTitleWithReadability - good content", () => { + const html = Deno.readTextFileSync("tests/test-data/regular-article.html"); + const result = extractTitleWithReadability(html); + assertEquals(result, "Article title here - ArticleParser"); +}); + +Deno.test("extractTitleWithReadability - page without title", () => { + const html = Deno.readTextFileSync("tests/test-data/html-no-title.html"); + const result = extractTitleWithReadability(html); + assertEquals(result, null); +}); + +Deno.test("extractTitleWithReadability - non-string", () => { + const result = extractTitleWithReadability({} as unknown as string); + assertEquals(result, null); +}); diff --git a/tests/helpers.ts b/tests/helpers.ts new file mode 100644 index 00000000..601436b5 --- /dev/null +++ b/tests/helpers.ts @@ -0,0 +1,15 @@ +// tests/helpers.ts + +export const createMockFetcher = ( + body: string, + contentType: string, + status = 200, +) => { + return () => + Promise.resolve( + new Response(body, { + status, + headers: { "content-type": contentType }, + }), + ); +}; diff --git a/tests/html.test.ts b/tests/html.test.ts new file mode 100644 index 00000000..bcdbf49f --- /dev/null +++ b/tests/html.test.ts @@ -0,0 +1,35 @@ +// tests/html.test.ts + +import { assertEquals } from "@std/assert"; +import { isString } from "@pwshub/bellajs"; +import { cleanify } from "../src/utils/html.ts"; +import { + defaultAllowedAttributes, + defaultAllowedIframeDomains, + defaultAllowedTags, +} from "../src/config.ts"; + +const defaults = { + allowedTags: defaultAllowedTags, + allowedAttributes: defaultAllowedAttributes, + allowedIframeDomains: defaultAllowedIframeDomains, +}; + +Deno.test("cleanify - removes unwanted elements/attributes", () => { + const html = Deno.readTextFileSync("tests/test-data/regular-article.html"); + assertEquals( + html.includes("
4746 Kelly Drive, West Virginia
"), + true, + ); + assertEquals( + html.includes(''), + true, + ); + const result = cleanify(html, defaults); + assertEquals(isString(result), true); + assertEquals( + result.includes("
4746 Kelly Drive, West Virginia
"), + false, + ); + assertEquals(result.includes('style="border: solid 1px #000"'), false); +}); diff --git a/tests/linker.test.ts b/tests/linker.test.ts new file mode 100644 index 00000000..12f7f2a4 --- /dev/null +++ b/tests/linker.test.ts @@ -0,0 +1,136 @@ +// tests/linker.test.ts + +import { assertEquals } from "@std/assert"; +import { isString } from "@pwshub/bellajs"; +import { + absolutify as absolutifyUrl, + chooseBestUrl, + isValid as isValidUrl, + normalize as normalizeUrls, + purify as purifyUrl, +} from "../src/utils/linker.ts"; + +Deno.test("isValidUrl - valid URLs", () => { + assertEquals(isValidUrl("https://www.23hq.com"), true); + assertEquals(isValidUrl("https://secure.actblue.com"), true); + assertEquals( + isValidUrl( + "https://docs.microsoft.com/en-us/azure/iot-edge/quickstart?view=iotedge-2018-06", + ), + true, + ); + assertEquals(isValidUrl("http://192.168.1.199:8081/example/page"), true); +}); + +Deno.test("isValidUrl - invalid URLs", () => { + assertEquals(isValidUrl("ftp://192.168.1.199:8081/example/page"), false); + assertEquals(isValidUrl(""), false); + assertEquals(isValidUrl(null as unknown as string), false); + assertEquals(isValidUrl({ a: "x" } as unknown as string), false); +}); + +Deno.test("normalizeUrls - adds absolute URLs and target=_blank", () => { + const bestUrl = "https://test-url.com/burritos-for-life"; + const html = Deno.readTextFileSync("tests/test-data/regular-article.html"); + const result = normalizeUrls(html, bestUrl); + assertEquals(isString(result), true); + assertEquals( + result.includes('watermelon'), + false, + ); + assertEquals( + result.includes( + 'watermelon', + ), + true, + ); + assertEquals( + result.includes( + 'rational peach', + ), + false, + ); + assertEquals( + result.includes( + 'rational peach', + ), + true, + ); +}); + +Deno.test("purifyUrl - various cases", () => { + assertEquals(purifyUrl(""), null); + assertEquals(purifyUrl({} as unknown as string), null); + assertEquals( + purifyUrl("https://some.where/article/abc-xyz"), + "https://some.where/article/abc-xyz", + ); + assertEquals( + purifyUrl("https://some.where/article/abc-xyz#name,bob"), + "https://some.where/article/abc-xyz", + ); + assertEquals( + purifyUrl( + "https://some.where/article/abc-xyz?utm_source=news4&utm_medium=email&utm_campaign=spring-summer", + ), + "https://some.where/article/abc-xyz", + ); + assertEquals( + purifyUrl( + "https://some.where/article/abc-xyz?q=3&utm_source=news4&utm_medium=email&utm_campaign=spring-summer", + ), + "https://some.where/article/abc-xyz?q=3", + ); + assertEquals( + purifyUrl( + "https://some.where/article/abc-xyz?pk_source=news4&pk_medium=email&pk_campaign=spring-summer", + ), + "https://some.where/article/abc-xyz", + ); + assertEquals( + purifyUrl( + "https://some.where/article/abc-xyz?q=3&pk_source=news4&pk_medium=email&pk_campaign=spring-summer", + ), + "https://some.where/article/abc-xyz?q=3", + ); +}); + +Deno.test("absolutifyUrl - various cases", () => { + assertEquals(absolutifyUrl("", ""), ""); + assertEquals(absolutifyUrl("", {} as unknown as string), ""); + assertEquals( + absolutifyUrl("https://some.where/article/abc-xyz", "category/page.html"), + "https://some.where/article/category/page.html", + ); + assertEquals( + absolutifyUrl( + "https://some.where/article/abc-xyz", + "../category/page.html", + ), + "https://some.where/category/page.html", + ); + assertEquals( + absolutifyUrl( + "https://some.where/blog/authors/article/abc-xyz", + "/category/page.html", + ), + "https://some.where/category/page.html", + ); + assertEquals( + absolutifyUrl("https://some.where/article/abc-xyz"), + "https://some.where/article/abc-xyz", + ); +}); + +Deno.test("chooseBestUrl - actual case", () => { + const title = "Google ฤ‘รฃ ra giรก mua Fitbit"; + const urls = [ + "https://alpha.xyz/tin-tuc-kinh-doanh/-/view_content/content/2965950/google-da-ra-gia-mua-fitbit", + "https://alpha.xyz/tin-tuc-kinh-doanh/view/2965950/907893219797", + "https://alpha.xyz/tin-tuc-kinh-doanh/google-da-ra-gia-mua-fitbit", + "https://a.xyz/read/google-da-ra-gia-mua-fitbit", + "https://a.xyz/read/2965950/907893219797", + ]; + const result = chooseBestUrl(urls, title); + assertEquals(result, urls[3]); +}); diff --git a/tests/main.test.ts b/tests/main.test.ts new file mode 100644 index 00000000..5c5fb65d --- /dev/null +++ b/tests/main.test.ts @@ -0,0 +1,105 @@ +// tests/main.test.ts + +import { assertEquals, assertRejects } from "@std/assert"; +import { extract } from "../src/main.ts"; +import { + defaultAllowedAttributes, + defaultAllowedIframeDomains, + defaultAllowedTags, +} from "../src/config.ts"; +import { createMockFetcher } from "./helpers.ts"; + +Deno.test("extract - non-string input", async () => { + await assertRejects( + () => extract({} as unknown as string), + Error, + "Input must be a string", + ); +}); + +Deno.test("extract - bad urls", async () => { + const badSamples = [ + "htt:/abc.com/failed-none-sense", + "fpt://abc.com/failed-none-sense", + "ttp://badcom/146753785", + "https://674458092126388225", + ]; + + for (const url of badSamples) { + await assertRejects( + () => extract(url), + Error, + ); + } +}); + +Deno.test("extract - regular article from url", async () => { + const expDesc = [ + "Navigation here Few can name a rational peach that isn't a conscientious goldfish!", + "One cannot separate snakes from plucky pomegranates?", + "Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs.", + ].join(" "); + + const html = Deno.readTextFileSync("tests/test-data/regular-article.html"); + const fetcher = createMockFetcher(html, "text/html"); + const result = await extract( + "https://somewhere.com/path/to/article", + {}, + fetcher, + ); + assertEquals((result as Record).title, "Article title here"); + assertEquals((result as Record).description, expDesc); +}); + +Deno.test("extract - no article from url", async () => { + const html = Deno.readTextFileSync("tests/test-data/html-no-article.html"); + const fetcher = createMockFetcher(html, "text/html"); + const result = await extract( + "https://somewhere.com/path/to/no/article", + {}, + fetcher, + ); + assertEquals(result, null); +}); + +Deno.test("extract - empty content from url", async () => { + const fetcher = createMockFetcher("", "text/html"); + const result = await extract( + "https://somewhere.com/path/to/no/content", + {}, + fetcher, + ); + assertEquals(result, null); +}); + +Deno.test("extract - html string directly", async () => { + const expDesc = [ + "Navigation here Few can name a rational peach that isn't a conscientious goldfish!", + "One cannot separate snakes from plucky pomegranates?", + "Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs.", + ].join(" "); + + const html = Deno.readTextFileSync("tests/test-data/regular-article.html"); + const result = await extract(html); + assertEquals((result as Record).title, "Article title here"); + assertEquals((result as Record).description, expDesc); +}); + +Deno.test("extract with custom allowed attributes via ParserOptions", async () => { + const html = Deno.readTextFileSync( + "tests/test-data/article-with-classes-attributes.html", + ); + const result = await extract(html, { + allowedTags: defaultAllowedTags, + allowedAttributes: { + ...defaultAllowedAttributes, + code: ["class"], + div: ["class"], + }, + allowedIframeDomains: defaultAllowedIframeDomains, + }); + assertEquals( + (result as Record).content.includes('code class="lang-js"'), + true, + ); +}); diff --git a/tests/parseFromHtml.test.ts b/tests/parseFromHtml.test.ts new file mode 100644 index 00000000..b61b25c2 --- /dev/null +++ b/tests/parseFromHtml.test.ts @@ -0,0 +1,146 @@ +// tests/parseFromHtml.test.ts + +import { assertEquals } from "@std/assert"; +import { extractFromHtml as parseFromHtml } from "../src/main.ts"; +import { + addTransformations, + removeTransformations, +} from "../src/utils/transformation.ts"; + +const expDesc = [ + "Navigation here Few can name a rational peach that isn't a conscientious goldfish!", + "One cannot separate snakes from plucky pomegranates?", + "Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs.", +].join(" "); + +Deno.test("parseFromHtml - webpage with no title", async () => { + const html = Deno.readTextFileSync("tests/test-data/html-no-title.html"); + const result = await parseFromHtml(html); + assertEquals(result, null); +}); + +Deno.test("parseFromHtml - webpage without link", async () => { + const html = Deno.readTextFileSync("tests/test-data/html-no-link.html"); + const result = await parseFromHtml(html); + assertEquals(result, null); +}); + +Deno.test("parseFromHtml - webpage with no main article", async () => { + const html = Deno.readTextFileSync("tests/test-data/html-no-article.html"); + const result = await parseFromHtml(html); + assertEquals(result, null); +}); + +Deno.test("parseFromHtml - webpage with very short article", async () => { + const html = Deno.readTextFileSync( + "tests/test-data/html-too-short-article.html", + ); + const result = await parseFromHtml(html, "abcd"); + assertEquals(result, null); +}); + +Deno.test("parseFromHtml - webpage with article but no source", async () => { + const html = Deno.readTextFileSync( + "tests/test-data/html-article-no-source.html", + ); + const result = await parseFromHtml(html); + assertEquals((result as Record).source, "somewhere.any"); +}); + +Deno.test("parseFromHtml - webpage with data-src in img tag", async () => { + const html = Deno.readTextFileSync( + "tests/test-data/html-article-with-data-src.html", + ); + const result = await parseFromHtml(html); + assertEquals( + (result as Record).content.includes( + '', + ), + true, + ); + assertEquals( + (result as Record).content.includes( + '', + ), + true, + ); +}); + +Deno.test("parseFromHtml - regular article", async () => { + const html = Deno.readTextFileSync("tests/test-data/regular-article.html"); + const result = await parseFromHtml( + html, + "https://somewhere.com/path/to/article", + ); + assertEquals((result as Record).title, "Article title here"); + assertEquals((result as Record).description, expDesc); + assertEquals( + (result as Record).content.includes( + '', + ), + true, + ); + assertEquals( + (result as Record).content.includes( + '', + ), + true, + ); +}); + +Deno.test("parseFromHtml - multi transforms", async () => { + addTransformations([ + { + patterns: [ + /http(s?):\/\/need-transform.tld\/*/, + ], + post: (document: Document) => { + document.querySelectorAll("a").forEach((node: Element) => { + const sHtml = node.innerHTML; + const link = node.getAttribute("href"); + node.parentNode!.replaceChild( + document.createTextNode(`[link url="${link}"]${sHtml}[/link]`), + node, + ); + }); + return document; + }, + }, + { + patterns: [ + /http(s?):\/\/sw.re\/*/, + ], + post: (document: Document) => { + document.querySelectorAll("strong").forEach((node: Element) => { + const b = document.createElement("B"); + b.innerHTML = node.innerHTML; + node.parentNode!.replaceChild(b, node); + }); + return document; + }, + }, + ]); + const html = Deno.readTextFileSync("tests/test-data/vnn-article.html"); + const url = "https://need-transform.tld/path/to/article"; + const result = await parseFromHtml(html, url); + assertEquals((result as Record).title, "Article title here"); + assertEquals( + (result as Record).content.includes( + '', + ), + false, + ); + assertEquals( + (result as Record).content.includes( + '[link url="https://vnn.vn/dict/watermelon"]watermelon[/link]', + ), + true, + ); + assertEquals( + (result as Record).content.includes( + "in its own way", + ), + true, + ); + removeTransformations(); +}); diff --git a/tests/retrieve.test.ts b/tests/retrieve.test.ts new file mode 100644 index 00000000..056c5458 --- /dev/null +++ b/tests/retrieve.test.ts @@ -0,0 +1,49 @@ +// tests/retrieve.test.ts + +import { assertEquals, assertRejects } from "@std/assert"; +import retrieve from "../src/utils/retrieve.ts"; +import { createMockFetcher } from "./helpers.ts"; + +Deno.test("retrieve - bad status code", async () => { + const url = "https://some.where/bad/page"; + const fetcher = createMockFetcher("Error 500", "text/plain", 500); + await assertRejects( + () => retrieve(url, fetcher), + Error, + "Request failed with error code 500", + ); +}); + +Deno.test("retrieve - good source", async () => { + const url = "https://some.where/good/page"; + const fetcher = createMockFetcher("
this is content
", "text/html"); + const buffer = await retrieve(url, fetcher); + const html = new TextDecoder().decode(buffer); + assertEquals(html, "
this is content
"); +}); + +Deno.test("retrieve - good source with \\r\\n", async () => { + const url = "https://some.where/good/page"; + const fetcher = createMockFetcher( + "\n\r\r\n\n
this is content
\n\r\r\n\n", + "text/html", + ); + const buffer = await retrieve(url, fetcher); + const html = new TextDecoder().decode(buffer).trim(); + assertEquals(html, "
this is content
"); +}); + +Deno.test("retrieve - custom fetcher", async () => { + const url = "https://some.where/good/source-with-fetcher"; + const innerFetcher = createMockFetcher( + "
this is content
", + "text/html", + ); + const myFetcher = (fetchUrl: string) => { + assertEquals(fetchUrl, url); + return innerFetcher(); + }; + const buffer = await retrieve(url, myFetcher); + const html = new TextDecoder().decode(buffer); + assertEquals(html, "
this is content
"); +}); diff --git a/test-data/article-with-classes-attributes.html b/tests/test-data/article-with-classes-attributes.html similarity index 58% rename from test-data/article-with-classes-attributes.html rename to tests/test-data/article-with-classes-attributes.html index 155dc66e..b415bcf8 100644 --- a/test-data/article-with-classes-attributes.html +++ b/tests/test-data/article-with-classes-attributes.html @@ -8,24 +8,33 @@ - + - + - - + + - - + + - - + + - + @@ -38,7 +47,8 @@

Article title here

-
Few can name a rational peach that isn't a conscientious goldfish! One cannot separate snakes from plucky pomegranates? Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs. Some posit the enchanting tiger to be less than confident. The literature would have us believe that an impartial turtle is not but a hippopotamus. Unfortunately, that is wrong; on the contrary, those cows are nothing more than pandas! The chicken is a shark; A turtle can hardly be considered a kind horse without also being a pomegranate. Zebras are witty persimmons.
+
Few can name a rational peach that isn't a conscientious goldfish! One cannot separate snakes from plucky pomegranates? Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs. Some posit the enchanting tiger to be less than confident. The literature would have us believe that an impartial turtle is not but a hippopotamus. Unfortunately, that is wrong; on the contrary, those cows are nothing more than pandas! The chicken is a shark; A turtle can hardly be considered a kind horse without also being a pomegranate. Zebras are witty persimmons.

Those cheetahs are nothing more than dogs. A watermelon is an exuberant kangaroo. An octopus is the tangerine of a grapes? The cherry is a shark. Recent controversy aside, they were lost without the cheerful plum that composed their fox. As far as we can estimate, one cannot separate camels from dynamic hamsters. Those tigers are nothing more than cows! A cow is a squirrel from the right perspective. Their banana was, in this moment, a helpful bear.

The first fair dog is, in its own way, a lemon.

diff --git a/test-data/html-article-no-source.html b/tests/test-data/html-article-no-source.html similarity index 87% rename from test-data/html-article-no-source.html rename to tests/test-data/html-article-no-source.html index 12237080..daee82c5 100644 --- a/test-data/html-article-no-source.html +++ b/tests/test-data/html-article-no-source.html @@ -1,13 +1,13 @@ - - - - - TechNews - - -
+ + + + + TechNews + + +
Few can name an easygoing deer that isn't a frank hamster. One cannot separate tangerines from excellent oranges. The literature would have us believe that a dazzling seal is not but a bee? Some posit the nice strawberry to be less than fair-minded. Washing and polishing the car,a monkey is the monkey of a turtle; What we don't know for sure is whether or not the spiders could be said to resemble ambitious cherries. Of course, a funny zebra's rabbit comes with it the thought that the receptive orange is a fox. If this was somewhat unclear, a sheep is a strawberry's chimpanzee. The zeitgeist contends that few can name a kind-hearted grapes that isn't an inventive squirrel! Few can name a friendly deer that isn't a decorous grapes? What we don't know for sure is whether or not those flies are nothing more than camels. @@ -18,5 +18,5 @@ Shouting with happiness, a courageous elephant is a duck of the mind? Some posit the upbeat hippopotamus to be less than enchanting. It's an undeniable fact, really; authors often misinterpret the grape as an endurable rabbit, when in actuality it feels more like a tough dolphin. We know that a cherry can hardly be considered a responsible apricot without also being a nectarine.
- + diff --git a/test-data/html-article-with-data-src.html b/tests/test-data/html-article-with-data-src.html similarity index 89% rename from test-data/html-article-with-data-src.html rename to tests/test-data/html-article-with-data-src.html index 9a51ac0b..b6359ff1 100644 --- a/test-data/html-article-with-data-src.html +++ b/tests/test-data/html-article-with-data-src.html @@ -1,13 +1,13 @@ - + - + Article title TechNews - - -
+ + +
Few can name an easygoing deer that isn't a frank hamster. One cannot separate tangerines from excellent oranges. @@ -15,6 +15,6 @@ The literature would have us believe that a dazzling seal is not but a bee? Some posit the nice strawberry to be less than fair-minded. Washing and polishing the car,a monkey is the monkey of a turtle; -
- +
+ diff --git a/test-data/html-no-article.html b/tests/test-data/html-no-article.html similarity index 69% rename from test-data/html-no-article.html rename to tests/test-data/html-no-article.html index 1212eccb..0da49e18 100644 --- a/test-data/html-no-article.html +++ b/tests/test-data/html-no-article.html @@ -1,8 +1,8 @@ - - - + + + Article title here - ArticleParser @@ -10,9 +10,9 @@ - + - - - + + diff --git a/test-data/html-no-link.html b/tests/test-data/html-no-link.html similarity index 68% rename from test-data/html-no-link.html rename to tests/test-data/html-no-link.html index 0d5f46d7..e7a59a56 100644 --- a/test-data/html-no-link.html +++ b/tests/test-data/html-no-link.html @@ -7,7 +7,8 @@ - + @@ -19,7 +20,8 @@

Article title here

-
Few can name a rational peach that isn't a conscientious goldfish! One cannot separate snakes from plucky pomegranates? Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs. Some posit the enchanting tiger to be less than confident. The literature would have us believe that an impartial turtle is not but a hippopotamus. Unfortunately, that is wrong; on the contrary, those cows are nothing more than pandas! The chicken is a shark; A turtle can hardly be considered a kind horse without also being a pomegranate. Zebras are witty persimmons.
+
Few can name a rational peach that isn't a conscientious goldfish! One cannot separate snakes from plucky pomegranates? Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs. Some posit the enchanting tiger to be less than confident. The literature would have us believe that an impartial turtle is not but a hippopotamus. Unfortunately, that is wrong; on the contrary, those cows are nothing more than pandas! The chicken is a shark; A turtle can hardly be considered a kind horse without also being a pomegranate. Zebras are witty persimmons.

Those cheetahs are nothing more than dogs. A watermelon is an exuberant kangaroo. An octopus is the tangerine of a grapes? The cherry is a shark. Recent controversy aside, they were lost without the cheerful plum that composed their fox. As far as we can estimate, one cannot separate camels from dynamic hamsters. Those tigers are nothing more than cows! A cow is a squirrel from the right perspective. Their banana was, in this moment, a helpful bear.

The first fair dog is, in its own way, a lemon.

diff --git a/test-data/html-no-title.html b/tests/test-data/html-no-title.html similarity index 59% rename from test-data/html-no-title.html rename to tests/test-data/html-no-title.html index f2f79321..c8e8efdb 100644 --- a/test-data/html-no-title.html +++ b/tests/test-data/html-no-title.html @@ -1,9 +1,9 @@ - - - - + + + +
Page header here
@@ -13,7 +13,8 @@

Article title here

-
Few can name a rational peach that isn't a conscientious goldfish! One cannot separate snakes from plucky pomegranates? Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs. Some posit the enchanting tiger to be less than confident. The literature would have us believe that an impartial turtle is not but a hippopotamus. Unfortunately, that is wrong; on the contrary, those cows are nothing more than pandas! The chicken is a shark; A turtle can hardly be considered a kind horse without also being a pomegranate. Zebras are witty persimmons.
+
Few can name a rational peach that isn't a conscientious goldfish! One cannot separate snakes from plucky pomegranates? Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs. Some posit the enchanting tiger to be less than confident. The literature would have us believe that an impartial turtle is not but a hippopotamus. Unfortunately, that is wrong; on the contrary, those cows are nothing more than pandas! The chicken is a shark; A turtle can hardly be considered a kind horse without also being a pomegranate. Zebras are witty persimmons.

Those cheetahs are nothing more than dogs. A watermelon is an exuberant kangaroo. An octopus is the tangerine of a grapes? The cherry is a shark. Recent controversy aside, they were lost without the cheerful plum that composed their fox. As far as we can estimate, one cannot separate camels from dynamic hamsters. Those tigers are nothing more than cows! A cow is a squirrel from the right perspective. Their banana was, in this moment, a helpful bear.

The first fair dog is, in its own way, a lemon.

diff --git a/tests/test-data/html-too-short-article.html b/tests/test-data/html-too-short-article.html new file mode 100644 index 00000000..9a7632ee --- /dev/null +++ b/tests/test-data/html-too-short-article.html @@ -0,0 +1,15 @@ + + + + + + + TechNews + + +
+ Hello world, this is too short, right? +

The first fair dog is, in its own way, a lemon.

+
+ + diff --git a/test-data/regular-article-date-itemprop.html b/tests/test-data/regular-article-date-itemprop.html similarity index 58% rename from test-data/regular-article-date-itemprop.html rename to tests/test-data/regular-article-date-itemprop.html index c23a4e9d..fcd5f55f 100644 --- a/test-data/regular-article-date-itemprop.html +++ b/tests/test-data/regular-article-date-itemprop.html @@ -8,23 +8,30 @@ - + - + - - + + - - + + - + @@ -37,9 +44,11 @@

Article title here

- + -
Few can name a rational peach that isn't a conscientious goldfish! One cannot separate snakes from plucky pomegranates? Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs. Some posit the enchanting tiger to be less than confident. The literature would have us believe that an impartial turtle is not but a hippopotamus. Unfortunately, that is wrong; on the contrary, those cows are nothing more than pandas! The chicken is a shark; A turtle can hardly be considered a kind horse without also being a pomegranate. Zebras are witty persimmons.
+
Few can name a rational peach that isn't a conscientious goldfish! One cannot separate snakes from plucky pomegranates? Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs. Some posit the enchanting tiger to be less than confident. The literature would have us believe that an impartial turtle is not but a hippopotamus. Unfortunately, that is wrong; on the contrary, those cows are nothing more than pandas! The chicken is a shark; A turtle can hardly be considered a kind horse without also being a pomegranate. Zebras are witty persimmons.

Those cheetahs are nothing more than dogs. A watermelon is an exuberant kangaroo. An octopus is the tangerine of a grapes? The cherry is a shark. Recent controversy aside, they were lost without the cheerful plum that composed their fox. As far as we can estimate, one cannot separate camels from dynamic hamsters. Those tigers are nothing more than cows! A cow is a squirrel from the right perspective. Their banana was, in this moment, a helpful bear.

The first fair dog is, in its own way, a lemon.

diff --git a/test-data/regular-article-date-span.html b/tests/test-data/regular-article-date-span.html similarity index 58% rename from test-data/regular-article-date-span.html rename to tests/test-data/regular-article-date-span.html index e6c13fc8..89e08b72 100644 --- a/test-data/regular-article-date-span.html +++ b/tests/test-data/regular-article-date-span.html @@ -8,23 +8,30 @@ - + - + - - + + - - + + - + @@ -37,9 +44,10 @@

Article title here

- Published at 11/09/2024 07h33 am + Published at 11/09/2024 07h33 am -
Few can name a rational peach that isn't a conscientious goldfish! One cannot separate snakes from plucky pomegranates? Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs. Some posit the enchanting tiger to be less than confident. The literature would have us believe that an impartial turtle is not but a hippopotamus. Unfortunately, that is wrong; on the contrary, those cows are nothing more than pandas! The chicken is a shark; A turtle can hardly be considered a kind horse without also being a pomegranate. Zebras are witty persimmons.
+
Few can name a rational peach that isn't a conscientious goldfish! One cannot separate snakes from plucky pomegranates? Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs. Some posit the enchanting tiger to be less than confident. The literature would have us believe that an impartial turtle is not but a hippopotamus. Unfortunately, that is wrong; on the contrary, those cows are nothing more than pandas! The chicken is a shark; A turtle can hardly be considered a kind horse without also being a pomegranate. Zebras are witty persimmons.

Those cheetahs are nothing more than dogs. A watermelon is an exuberant kangaroo. An octopus is the tangerine of a grapes? The cherry is a shark. Recent controversy aside, they were lost without the cheerful plum that composed their fox. As far as we can estimate, one cannot separate camels from dynamic hamsters. Those tigers are nothing more than cows! A cow is a squirrel from the right perspective. Their banana was, in this moment, a helpful bear.

The first fair dog is, in its own way, a lemon.

diff --git a/test-data/regular-article-date-time.html b/tests/test-data/regular-article-date-time.html similarity index 57% rename from test-data/regular-article-date-time.html rename to tests/test-data/regular-article-date-time.html index d1e1638f..6b32cd20 100644 --- a/test-data/regular-article-date-time.html +++ b/tests/test-data/regular-article-date-time.html @@ -8,23 +8,30 @@ - + - + - - + + - - + + - + @@ -37,9 +44,11 @@

Article title here

- + -
Few can name a rational peach that isn't a conscientious goldfish! One cannot separate snakes from plucky pomegranates? Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs. Some posit the enchanting tiger to be less than confident. The literature would have us believe that an impartial turtle is not but a hippopotamus. Unfortunately, that is wrong; on the contrary, those cows are nothing more than pandas! The chicken is a shark; A turtle can hardly be considered a kind horse without also being a pomegranate. Zebras are witty persimmons.
+
Few can name a rational peach that isn't a conscientious goldfish! One cannot separate snakes from plucky pomegranates? Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs. Some posit the enchanting tiger to be less than confident. The literature would have us believe that an impartial turtle is not but a hippopotamus. Unfortunately, that is wrong; on the contrary, those cows are nothing more than pandas! The chicken is a shark; A turtle can hardly be considered a kind horse without also being a pomegranate. Zebras are witty persimmons.

Those cheetahs are nothing more than dogs. A watermelon is an exuberant kangaroo. An octopus is the tangerine of a grapes? The cherry is a shark. Recent controversy aside, they were lost without the cheerful plum that composed their fox. As far as we can estimate, one cannot separate camels from dynamic hamsters. Those tigers are nothing more than cows! A cow is a squirrel from the right perspective. Their banana was, in this moment, a helpful bear.

The first fair dog is, in its own way, a lemon.

diff --git a/test-data/regular-article-json-ld.html b/tests/test-data/regular-article-json-ld.html similarity index 58% rename from test-data/regular-article-json-ld.html rename to tests/test-data/regular-article-json-ld.html index c48ad9db..a4960d75 100644 --- a/test-data/regular-article-json-ld.html +++ b/tests/test-data/regular-article-json-ld.html @@ -6,36 +6,43 @@ Article title here - ArticleParser - + - + - + } + ] + - + - - + + - + @@ -70,7 +80,8 @@

Article title here

-
Few can name a rational peach that isn't a conscientious goldfish! One cannot separate snakes from plucky pomegranates? Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs. Some posit the enchanting tiger to be less than confident. The literature would have us believe that an impartial turtle is not but a hippopotamus. Unfortunately, that is wrong; on the contrary, those cows are nothing more than pandas! The chicken is a shark; A turtle can hardly be considered a kind horse without also being a pomegranate. Zebras are witty persimmons.
+
Few can name a rational peach that isn't a conscientious goldfish! One cannot separate snakes from plucky pomegranates? Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs. Some posit the enchanting tiger to be less than confident. The literature would have us believe that an impartial turtle is not but a hippopotamus. Unfortunately, that is wrong; on the contrary, those cows are nothing more than pandas! The chicken is a shark; A turtle can hardly be considered a kind horse without also being a pomegranate. Zebras are witty persimmons.

Those cheetahs are nothing more than dogs. A watermelon is an exuberant kangaroo. An octopus is the tangerine of a grapes? The cherry is a shark. Recent controversy aside, they were lost without the cheerful plum that composed their fox. As far as we can estimate, one cannot separate camels from dynamic hamsters. Those tigers are nothing more than cows! A cow is a squirrel from the right perspective. Their banana was, in this moment, a helpful bear.

The first fair dog is, in its own way, a lemon.

diff --git a/test-data/regular-article.html b/tests/test-data/regular-article.html similarity index 56% rename from test-data/regular-article.html rename to tests/test-data/regular-article.html index ab68045c..0749b7e7 100644 --- a/test-data/regular-article.html +++ b/tests/test-data/regular-article.html @@ -8,25 +8,34 @@ - + - + - - + + - - + + - - + + - + @@ -39,7 +48,8 @@

Article title here

-
Few can name a rational peach that isn't a conscientious goldfish! One cannot separate snakes from plucky pomegranates? Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs. Some posit the enchanting tiger to be less than confident. The literature would have us believe that an impartial turtle is not but a hippopotamus. Unfortunately, that is wrong; on the contrary, those cows are nothing more than pandas! The chicken is a shark; A turtle can hardly be considered a kind horse without also being a pomegranate. Zebras are witty persimmons.
+
Few can name a rational peach that isn't a conscientious goldfish! One cannot separate snakes from plucky pomegranates? Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs. Some posit the enchanting tiger to be less than confident. The literature would have us believe that an impartial turtle is not but a hippopotamus. Unfortunately, that is wrong; on the contrary, those cows are nothing more than pandas! The chicken is a shark; A turtle can hardly be considered a kind horse without also being a pomegranate. Zebras are witty persimmons.

Those cheetahs are nothing more than dogs. A watermelon is an exuberant kangaroo. An octopus is the tangerine of a grapes? The cherry is a shark. Recent controversy aside, they were lost without the cheerful plum that composed their fox. As far as we can estimate, one cannot separate camels from dynamic hamsters. Those tigers are nothing more than cows! A cow is a squirrel from the right perspective. Their banana was, in this moment, a helpful bear.

The first fair dog is, in its own way, a lemon.

diff --git a/test-data/vnn-article.html b/tests/test-data/vnn-article.html similarity index 64% rename from test-data/vnn-article.html rename to tests/test-data/vnn-article.html index 41ab9350..65a875a9 100644 --- a/test-data/vnn-article.html +++ b/tests/test-data/vnn-article.html @@ -8,24 +8,31 @@ - + - + - - + + - - + + - + @@ -38,7 +45,8 @@

Article title here

-
Few can name a rational peach that isn't a conscientious goldfish! One cannot separate snakes from plucky pomegranates? Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs. Some posit the enchanting tiger to be less than confident. The literature would have us believe that an impartial turtle is not but a hippopotamus. Unfortunately, that is wrong; on the contrary, those cows are nothing more than pandas! The chicken is a shark; A turtle can hardly be considered a kind horse without also being a pomegranate. Zebras are witty persimmons.
+
Few can name a rational peach that isn't a conscientious goldfish! One cannot separate snakes from plucky pomegranates? Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs. Some posit the enchanting tiger to be less than confident. The literature would have us believe that an impartial turtle is not but a hippopotamus. Unfortunately, that is wrong; on the contrary, those cows are nothing more than pandas! The chicken is a shark; A turtle can hardly be considered a kind horse without also being a pomegranate. Zebras are witty persimmons.

Those cheetahs are nothing more than dogs. A watermelon is an exuberant kangaroo. An octopus is the tangerine of a grapes? The cherry is a shark. Recent controversy aside, they were lost without the cheerful plum that composed their fox. As far as we can estimate, one cannot separate camels from dynamic hamsters. Those tigers are nothing more than cows! A cow is a squirrel from the right perspective. Their banana was, in this moment, a helpful bear.

diff --git a/tests/transformation.test.ts b/tests/transformation.test.ts new file mode 100644 index 00000000..2e9202d3 --- /dev/null +++ b/tests/transformation.test.ts @@ -0,0 +1,203 @@ +// tests/transformation.test.ts + +import { assertEquals } from "@std/assert"; +import { + addTransformations, + execPostParser, + execPreParser, + findTransformations, + getTransformations, + removeTransformations, +} from "../src/utils/transformation.ts"; + +Deno.test("add one transformation object", () => { + const result = addTransformations({ + patterns: [ + /http(s?):\/\/([\w]+.)?def.tld\/*/, + ], + pre: (document: Document) => { + return document; + }, + post: (document: Document) => { + return document; + }, + }); + assertEquals(result, 1); +}); + +Deno.test("add multi transformation object", () => { + const result = addTransformations([ + { + patterns: [ + /http(s?):\/\/google.com\/*/, + /http(s?):\/\/goo.gl\/*/, + ], + }, + { + patterns: [ + /http(s?):\/\/goo.gl\/*/, + /http(s?):\/\/google.inc\/*/, + ], + }, + ]); + assertEquals(result, 2); +}); + +Deno.test("add transformation object without patterns", () => { + const result = addTransformations({ + pre: (document: Document) => { + return document; + }, + post: (document: Document) => { + return document; + }, + } as any); + assertEquals(result, 0); +}); + +Deno.test("add transformation object without valid patterns", () => { + const result = addTransformations({ + patterns: 123 as any, + pre: (document: Document) => { + return document; + }, + post: (document: Document) => { + return document; + }, + }); + assertEquals(result, 0); +}); + +Deno.test("get all transformations", () => { + const result = getTransformations(); + assertEquals(result.length, 3); + assertEquals( + String(result[0].patterns[0]), + String(/http(s?):\/\/([\w]+.)?def.tld\/*/), + ); +}); + +Deno.test("remove one transformation", () => { + addTransformations([ + { + patterns: [ + /http(s?):\/\/abc.com\/*/, + /http(s?):\/\/def.gl\/*/, + ], + }, + { + patterns: [ + /http(s?):\/\/hik.gl\/*/, + /http(s?):\/\/lmn.inc\/*/, + ], + }, + { + patterns: [ + /http(s?):\/\/opq.gl\/*/, + /http(s?):\/\/rst.inc\/*/, + ], + }, + ]); + const result = removeTransformations([ + /http(s?):\/\/goo.gl\/*/, + ]); + assertEquals(result, 2); +}); + +Deno.test("get all transformations again", () => { + const result = getTransformations(); + assertEquals(result.length, 4); + assertEquals( + String(result[3].patterns[1]), + String(/http(s?):\/\/rst.inc\/*/), + ); +}); + +Deno.test("find transformations", () => { + addTransformations([ + { + patterns: [ + /http(s?):\/\/def.gl\/*/, + /http(s?):\/\/uvw.inc\/*/, + ], + }, + ]); + const notFound = findTransformations([ + "https://goo.gl/docs/article.html", + ]); + assertEquals(notFound.length, 0); + + const foundOne = findTransformations([ + "https://lmn.inc/docs/article.html", + ]); + assertEquals(foundOne.length, 1); + + const foundTwo = findTransformations([ + "https://def.gl/docs/article.html", + ]); + assertEquals(foundTwo.length, 2); +}); + +Deno.test("execPreParser", () => { + addTransformations([ + { + patterns: [ + /http(s?):\/\/xyz.com\/*/, + ], + pre: (doc: Document) => { + doc.querySelectorAll(".adv").forEach((element: Element) => { + element.parentNode!.removeChild(element); + }); + return doc; + }, + }, + ]); + const html = ` +
+ hi user, this is an advertisement element +
free product now!
+
+ `; + const result = execPreParser(html, "https://xyz.com/article"); + assertEquals( + result.includes("hi user, this is an advertisement element"), + true, + ); + assertEquals( + result.includes('
free product now!
'), + false, + ); +}); + +Deno.test("execPostParser", () => { + addTransformations([ + { + patterns: [ + /http(s?):\/\/xyz.com\/*/, + ], + post: (doc: Document) => { + doc.querySelectorAll("b").forEach((element: Element) => { + const itag = doc.createElement("i"); + itag.innerHTML = element.innerHTML; + element.parentNode!.replaceChild(itag, element); + }); + return doc; + }, + }, + ]); + const html = ` +
+ hi user, +

Thank you for your feedback!

+
+ `; + const result = execPostParser(html, "https://xyz.com/article"); + assertEquals(result.includes("user"), true); + assertEquals(result.includes("user"), false); +}); + +Deno.test("remove all transformations", () => { + const result = removeTransformations(); + assertEquals(result, 7); + assertEquals(getTransformations().length, 0); +});