v9.0.1-rc2

deno.json

@@ -1,6 +1,6 @@
 {
   "name": "@extractus/article-extractor",
-  "version": "9.0.1-rc1",
+  "version": "9.0.1-rc2",
   "description": "To extract main article from given URL",
   "homepage": "https://github.com/extractus/article-extractor",
   "repository": {

src/config.ts

@@ -78,12 +78,12 @@ export const defaultAllowedAttributes: Record<string, string[]> = {
 export const defaultAllowedIframeDomains: string[] = [
   "youtube.com",
   "vimeo.com",
-  "odysee.com",
   "soundcloud.com",
   "audius.co",
   "github.com",
-  "codepen.com",
   "twitter.com",
+  "x.com",
+  "tiktok.com",
   "facebook.com",
   "instagram.com",
 ];

src/main.ts

@@ -5,64 +5,7 @@ import parseFromHtml from "./utils/parseFromHtml.ts";
 import { getCharset, isHTML } from "./utils/html.ts";
 import { isValid as isValidUrl } from "./utils/linker.ts";
 
-/** Custom fetch function signature. */
-export type Fetcher = (url: string) => Promise<Response>;
-
-/** Per-site HTML pre/post processing transformation. */
-export interface Transformation {
-  /** URL regex patterns to match against article links. */
-  patterns: RegExp[];
-  /** Function to pre-process raw HTML before extraction. */
-  pre?: (document: Document) => Document;
-  /** Function to post-process extracted article content. */
-  post?: (document: Document) => Document;
-}
-
-/** Options for the article extraction process. */
-export interface ParserOptions {
-  /** Words per minute for time-to-read estimation. Default: 300 */
-  wordsPerMinute?: number;
-  /** Max characters for generated description. Default: 210 */
-  descriptionTruncateLen?: number;
-  /** Min characters required to keep meta description. Default: 180 */
-  descriptionLengthThreshold?: number;
-  /** Min characters required for article content. Default: 200 */
-  contentLengthThreshold?: number;
-  /** Allowed HTML tags in output. Default: list of semantic/content tags. */
-  allowedTags?: string[];
-  /** Per-tag allowed attributes. Default: src, href, alt, etc. */
-  allowedAttributes?: Record<string, string[]>;
-  /** Allowed domains for iframe src. Default: youtube, vimeo, etc. */
-  allowedIframeDomains?: string[];
-}
-
-/** Extracted article data structure. */
-export interface ArticleData {
-  /** Best resolved URL of the article. */
-  url?: string;
-  /** Alternative URLs discovered (canonical, shortlink, amphtml). */
-  links?: string[];
-  /** Article title. */
-  title?: string;
-  /** Short description or excerpt. */
-  description?: string;
-  /** Main image URL. */
-  image?: string;
-  /** Site favicon URL. */
-  favicon?: string;
-  /** Author name. */
-  author?: string;
-  /** Extracted article HTML content. */
-  content?: string;
-  /** Original publisher/source domain. */
-  source?: string;
-  /** Publication date string. */
-  published?: string;
-  /** Estimated time to read in seconds. */
-  ttr?: number;
-  /** Page type (e.g. "article"). */
-  type?: string;
-}
+import type { ArticleData, Fetcher, ParserOptions } from "./types.ts";
 
 /**
  * Load and extract article data from a URL or HTML string.
@@ -123,3 +66,5 @@ export {
   addTransformations,
   removeTransformations,
 } from "./utils/transformation.ts";
+
+export * from "./types.ts";

src/types.ts

@@ -0,0 +1,60 @@
+// types.ts
+
+/** Custom fetch function signature. */
+export type Fetcher = (url: string) => Promise<Response>;
+
+/** Per-site HTML pre/post processing transformation. */
+export type Transformation = {
+  /** URL regex patterns to match against article links. */
+  patterns: RegExp[];
+  /** Function to pre-process raw HTML before extraction. */
+  pre?: (document: Document) => Document;
+  /** Function to post-process extracted article content. */
+  post?: (document: Document) => Document;
+};
+
+/** Options for the article extraction process. */
+export type ParserOptions = {
+  /** Words per minute for time-to-read estimation. Default: 300 */
+  wordsPerMinute?: number;
+  /** Max characters for generated description. Default: 210 */
+  descriptionTruncateLen?: number;
+  /** Min characters required to keep meta description. Default: 180 */
+  descriptionLengthThreshold?: number;
+  /** Min characters required for article content. Default: 200 */
+  contentLengthThreshold?: number;
+  /** Allowed HTML tags in output. Default: list of semantic/content tags. */
+  allowedTags?: string[];
+  /** Per-tag allowed attributes. Default: src, href, alt, etc. */
+  allowedAttributes?: Record<string, string[]>;
+  /** Allowed domains for iframe src. Default: youtube, vimeo, etc. */
+  allowedIframeDomains?: string[];
+};
+
+/** Extracted article data structure. */
+export type ArticleData = {
+  /** Best resolved URL of the article. */
+  url?: string;
+  /** Alternative URLs discovered (canonical, shortlink, amphtml). */
+  links?: string[];
+  /** Article title. */
+  title?: string;
+  /** Short description or excerpt. */
+  description?: string;
+  /** Main image URL. */
+  image?: string;
+  /** Site favicon URL. */
+  favicon?: string;
+  /** Author name. */
+  author?: string;
+  /** Extracted article HTML content. */
+  content?: string;
+  /** Original publisher/source domain. */
+  source?: string;
+  /** Publication date string. */
+  published?: string;
+  /** Estimated time to read in seconds. */
+  ttr?: number;
+  /** Page type (e.g. "article"). */
+  type?: string;
+};

src/utils/parseFromHtml.ts

@@ -25,7 +25,15 @@ import {
   defaultAllowedTags,
 } from "../config.ts";
 
-import type { ArticleData, ParserOptions } from "../main.ts";
+import type { ArticleData, ParserOptions } from "../types.ts";
+
+const fmtIso = (input: string): string => {
+  try {
+    return (new Date(input)).toISOString();
+  } catch {
+    return input;
+  }
+};
 
 const summarize = (
   { desc, text, threshold, maxlen }: {
@@ -139,7 +147,7 @@ export default async (
     author: author || readabilityExtracted.byline || "",
     favicon,
     source: readabilityExtracted.siteName || getDomain(bestUrl),
-    published: readabilityExtracted.publishedTime || published,
+    published: fmtIso(readabilityExtracted.publishedTime || published),
     ttr: getTTR(textContent, imgcount, wordsPerMinute),
     type,
   };

src/utils/similarity.ts

@@ -2,16 +2,16 @@
 
 import { compareTwoStrings, isArray, isString } from "@pwshub/bellajs";
 
-interface MatchResult {
+type MatchResult = {
   target: string;
   rating: number;
-}
+};
 
-interface BestMatchResult {
+type BestMatchResult = {
   ratings: MatchResult[];
   bestMatch: MatchResult;
   bestMatchIndex: number;
-}
+};
 
 const areArgsValid = (mainString: unknown, targetStrings: unknown): boolean => {
   return isString(mainString) && isArray(targetStrings) &&

src/utils/transformation.ts

@@ -3,11 +3,7 @@
 import { isArray, isFunction } from "@pwshub/bellajs";
 import { DOMParser } from "linkedom";
 
-export interface Transformation {
-  patterns: RegExp[];
-  pre?: (document: Document) => Document;
-  post?: (document: Document) => Document;
-}
+import type { Transformation } from "../types.ts";
 
 const transformations: Transformation[] = [];