From e9f7b2e2191157d16bc56d15da691ddb23937346 Mon Sep 17 00:00:00 2001 From: Dong Nguyen Date: Thu, 20 Aug 2026 15:28:33 +0700 Subject: [PATCH] v9.0.1-rc2 - Improve types --- deno.json | 2 +- src/config.ts | 4 +-- src/main.ts | 61 ++----------------------------------- src/types.ts | 60 ++++++++++++++++++++++++++++++++++++ src/utils/parseFromHtml.ts | 12 ++++++-- src/utils/similarity.ts | 8 ++--- src/utils/transformation.ts | 6 +--- 7 files changed, 81 insertions(+), 72 deletions(-) create mode 100644 src/types.ts diff --git a/deno.json b/deno.json index 186c8a7..46bc08d 100644 --- a/deno.json +++ b/deno.json @@ -1,6 +1,6 @@ { "name": "@extractus/article-extractor", - "version": "9.0.1-rc1", + "version": "9.0.1-rc2", "description": "To extract main article from given URL", "homepage": "https://github.com/extractus/article-extractor", "repository": { diff --git a/src/config.ts b/src/config.ts index 1504747..20c2ff3 100644 --- a/src/config.ts +++ b/src/config.ts @@ -78,12 +78,12 @@ export const defaultAllowedAttributes: Record = { export const defaultAllowedIframeDomains: string[] = [ "youtube.com", "vimeo.com", - "odysee.com", "soundcloud.com", "audius.co", "github.com", - "codepen.com", "twitter.com", + "x.com", + "tiktok.com", "facebook.com", "instagram.com", ]; diff --git a/src/main.ts b/src/main.ts index bc4a9d5..4164731 100644 --- a/src/main.ts +++ b/src/main.ts @@ -5,64 +5,7 @@ import parseFromHtml from "./utils/parseFromHtml.ts"; import { getCharset, isHTML } from "./utils/html.ts"; import { isValid as isValidUrl } from "./utils/linker.ts"; -/** Custom fetch function signature. */ -export type Fetcher = (url: string) => Promise; - -/** Per-site HTML pre/post processing transformation. */ -export interface Transformation { - /** URL regex patterns to match against article links. */ - patterns: RegExp[]; - /** Function to pre-process raw HTML before extraction. */ - pre?: (document: Document) => Document; - /** Function to post-process extracted article content. */ - post?: (document: Document) => Document; -} - -/** Options for the article extraction process. */ -export interface ParserOptions { - /** Words per minute for time-to-read estimation. Default: 300 */ - wordsPerMinute?: number; - /** Max characters for generated description. Default: 210 */ - descriptionTruncateLen?: number; - /** Min characters required to keep meta description. Default: 180 */ - descriptionLengthThreshold?: number; - /** Min characters required for article content. Default: 200 */ - contentLengthThreshold?: number; - /** Allowed HTML tags in output. Default: list of semantic/content tags. */ - allowedTags?: string[]; - /** Per-tag allowed attributes. Default: src, href, alt, etc. */ - allowedAttributes?: Record; - /** Allowed domains for iframe src. Default: youtube, vimeo, etc. */ - allowedIframeDomains?: string[]; -} - -/** Extracted article data structure. */ -export interface ArticleData { - /** Best resolved URL of the article. */ - url?: string; - /** Alternative URLs discovered (canonical, shortlink, amphtml). */ - links?: string[]; - /** Article title. */ - title?: string; - /** Short description or excerpt. */ - description?: string; - /** Main image URL. */ - image?: string; - /** Site favicon URL. */ - favicon?: string; - /** Author name. */ - author?: string; - /** Extracted article HTML content. */ - content?: string; - /** Original publisher/source domain. */ - source?: string; - /** Publication date string. */ - published?: string; - /** Estimated time to read in seconds. */ - ttr?: number; - /** Page type (e.g. "article"). */ - type?: string; -} +import type { ArticleData, Fetcher, ParserOptions } from "./types.ts"; /** * Load and extract article data from a URL or HTML string. @@ -123,3 +66,5 @@ export { addTransformations, removeTransformations, } from "./utils/transformation.ts"; + +export * from "./types.ts"; diff --git a/src/types.ts b/src/types.ts new file mode 100644 index 0000000..43b7028 --- /dev/null +++ b/src/types.ts @@ -0,0 +1,60 @@ +// types.ts + +/** Custom fetch function signature. */ +export type Fetcher = (url: string) => Promise; + +/** Per-site HTML pre/post processing transformation. */ +export type Transformation = { + /** URL regex patterns to match against article links. */ + patterns: RegExp[]; + /** Function to pre-process raw HTML before extraction. */ + pre?: (document: Document) => Document; + /** Function to post-process extracted article content. */ + post?: (document: Document) => Document; +}; + +/** Options for the article extraction process. */ +export type ParserOptions = { + /** Words per minute for time-to-read estimation. Default: 300 */ + wordsPerMinute?: number; + /** Max characters for generated description. Default: 210 */ + descriptionTruncateLen?: number; + /** Min characters required to keep meta description. Default: 180 */ + descriptionLengthThreshold?: number; + /** Min characters required for article content. Default: 200 */ + contentLengthThreshold?: number; + /** Allowed HTML tags in output. Default: list of semantic/content tags. */ + allowedTags?: string[]; + /** Per-tag allowed attributes. Default: src, href, alt, etc. */ + allowedAttributes?: Record; + /** Allowed domains for iframe src. Default: youtube, vimeo, etc. */ + allowedIframeDomains?: string[]; +}; + +/** Extracted article data structure. */ +export type ArticleData = { + /** Best resolved URL of the article. */ + url?: string; + /** Alternative URLs discovered (canonical, shortlink, amphtml). */ + links?: string[]; + /** Article title. */ + title?: string; + /** Short description or excerpt. */ + description?: string; + /** Main image URL. */ + image?: string; + /** Site favicon URL. */ + favicon?: string; + /** Author name. */ + author?: string; + /** Extracted article HTML content. */ + content?: string; + /** Original publisher/source domain. */ + source?: string; + /** Publication date string. */ + published?: string; + /** Estimated time to read in seconds. */ + ttr?: number; + /** Page type (e.g. "article"). */ + type?: string; +}; diff --git a/src/utils/parseFromHtml.ts b/src/utils/parseFromHtml.ts index 04a330f..c0f485a 100644 --- a/src/utils/parseFromHtml.ts +++ b/src/utils/parseFromHtml.ts @@ -25,7 +25,15 @@ import { defaultAllowedTags, } from "../config.ts"; -import type { ArticleData, ParserOptions } from "../main.ts"; +import type { ArticleData, ParserOptions } from "../types.ts"; + +const fmtIso = (input: string): string => { + try { + return (new Date(input)).toISOString(); + } catch { + return input; + } +}; const summarize = ( { desc, text, threshold, maxlen }: { @@ -139,7 +147,7 @@ export default async ( author: author || readabilityExtracted.byline || "", favicon, source: readabilityExtracted.siteName || getDomain(bestUrl), - published: readabilityExtracted.publishedTime || published, + published: fmtIso(readabilityExtracted.publishedTime || published), ttr: getTTR(textContent, imgcount, wordsPerMinute), type, }; diff --git a/src/utils/similarity.ts b/src/utils/similarity.ts index e25ff11..4b4e3c4 100644 --- a/src/utils/similarity.ts +++ b/src/utils/similarity.ts @@ -2,16 +2,16 @@ import { compareTwoStrings, isArray, isString } from "@pwshub/bellajs"; -interface MatchResult { +type MatchResult = { target: string; rating: number; -} +}; -interface BestMatchResult { +type BestMatchResult = { ratings: MatchResult[]; bestMatch: MatchResult; bestMatchIndex: number; -} +}; const areArgsValid = (mainString: unknown, targetStrings: unknown): boolean => { return isString(mainString) && isArray(targetStrings) && diff --git a/src/utils/transformation.ts b/src/utils/transformation.ts index 35793e5..0bb4864 100644 --- a/src/utils/transformation.ts +++ b/src/utils/transformation.ts @@ -3,11 +3,7 @@ import { isArray, isFunction } from "@pwshub/bellajs"; import { DOMParser } from "linkedom"; -export interface Transformation { - patterns: RegExp[]; - pre?: (document: Document) => Document; - post?: (document: Document) => Document; -} +import type { Transformation } from "../types.ts"; const transformations: Transformation[] = [];