From c82b1ac6b025178a52ca632b5d3d49b8ff737a2f Mon Sep 17 00:00:00 2001 From: murdle Date: Tue, 23 Dec 2025 02:14:20 +0200 Subject: [PATCH] add feed --- .gitignore | 3 + README.md | 2 +- src/api/channel.ts | 45 ++----------- src/api/feed.ts | 25 +++++++ src/api/video.ts | 35 +++++----- src/index.ts | 2 + src/lib/channel.ts | 16 ----- src/lib/config.ts | 21 ++++-- src/lib/innertube.ts | 11 +++- src/lib/utils.ts | 66 +++++++++++++++++++ src/lib/utils/channel.ts | 42 ++++++++++++ src/lib/utils/feed.ts | 74 +++++++++++++++++++++ src/lib/utils/video.ts | 40 ++++++++++++ src/models/FeedGetSchema.ts | 6 ++ src/templates/channelData.ts | 27 ++++---- src/templates/homeFeed.ts | 106 ++++++++++++++++++++++++++++++ src/templates/search.ts | 122 +++++++++++++++++------------------ 17 files changed, 490 insertions(+), 153 deletions(-) create mode 100644 src/api/feed.ts delete mode 100644 src/lib/channel.ts create mode 100644 src/lib/utils.ts create mode 100644 src/lib/utils/channel.ts create mode 100644 src/lib/utils/feed.ts create mode 100644 src/lib/utils/video.ts create mode 100644 src/models/FeedGetSchema.ts create mode 100644 src/templates/homeFeed.ts diff --git a/.gitignore b/.gitignore index 36fabb6..870978f 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,6 @@ lerna-debug.log* # misc .DS_Store + +# cache +cache/ \ No newline at end of file diff --git a/README.md b/README.md index e12b31d..65b7eb7 100644 --- a/README.md +++ b/README.md @@ -4,5 +4,5 @@ npm run dev ``` ``` -open http://localhost:3000 +open http://localhost:4000 ``` diff --git a/src/api/channel.ts b/src/api/channel.ts index c132c50..dc73e33 100644 --- a/src/api/channel.ts +++ b/src/api/channel.ts @@ -1,10 +1,9 @@ import { zValidator } from "@hono/zod-validator"; import innertube from "../lib/innertube.js"; -import { ChannelFindSchema } from "../models/ChannelFindSchema.js"; -import { Hono } from "hono"; import channelData from "../templates/channelData.js"; -import { parseSubscriberCount } from "../lib/channel.js"; -import { YTNodes } from "youtubei.js"; +import { ChannelFindSchema } from "../models/ChannelFindSchema.js"; +import { parseChannel } from "../lib/utils/channel.js"; +import { Hono } from "hono"; const channel = new Hono(); @@ -16,45 +15,13 @@ channel.get( try { const data = await innertube.getChannel(id); - const { header } = data; + const info = parseChannel(data); - if (!header) { + if (!info) { return c.text("Channel not found", 404); } - const channelInfo = header.is(YTNodes.PageHeader) - ? { - name: header.page_title, - description: header.content?.description?.description?.text || "", - thumbnail: (header.content?.image as YTNodes.DecoratedAvatarView) - ?.avatar?.image?.[0]?.url || "", - subCount: parseSubscriberCount( - header.content?.metadata?.metadata_rows?.[1] - ?.metadata_parts?.[0]?.text?.text || "0" - ), - } - : header.is(YTNodes.C4TabbedHeader) - ? { - name: header.author?.name || "", - description: data.metadata?.description || "", - thumbnail: header.author?.best_thumbnail?.url || "", - subCount: parseSubscriberCount(header.subscribers?.text || "0"), - } - : null; - - if (!channelInfo) { - return c.text("Channel not found", 404); - } - - return c.html( - channelData( - id, - channelInfo.name, - channelInfo.subCount, - channelInfo.description, - channelInfo.thumbnail - ) - ); + return c.html(channelData(info)); } catch (err) { console.error("Error fetching channel:", err); return c.text("Channel not found", 404); diff --git a/src/api/feed.ts b/src/api/feed.ts new file mode 100644 index 0000000..1bb3f69 --- /dev/null +++ b/src/api/feed.ts @@ -0,0 +1,25 @@ +import { zValidator } from "@hono/zod-validator"; +import { Hono } from "hono"; +import innertube from "../lib/innertube.js"; +import homeFeed from "../templates/homeFeed.js"; +import { FeedGetSchema } from "../models/FeedGetSchema.js"; +import { parseHomeFeed } from "../lib/utils/feed.js"; +import type { YT } from "youtubei.js"; + +const feed = new Hono(); + +feed.get( + "/standardfeeds/:region/:type", + zValidator("param", FeedGetSchema), + async (c) => { + const { type } = c.req.valid("param"); + if (type != "most_popular") return c.html(homeFeed()) + + const data = await innertube.getHomeFeed(); + const feed = data as YT.HomeFeed; + + return c.html(homeFeed(await parseHomeFeed(feed))) + } +) + +export default feed \ No newline at end of file diff --git a/src/api/video.ts b/src/api/video.ts index 1eb481c..bd16a38 100644 --- a/src/api/video.ts +++ b/src/api/video.ts @@ -3,6 +3,7 @@ import innertube from "../lib/innertube.js"; import { zValidator } from "@hono/zod-validator"; import { VideoSearchSchema } from "../models/VideoSearchSchema.js"; import search from "../templates/search.js"; +import { parseVideos } from "../lib/utils/video.js"; import { YTNodes } from "youtubei.js"; const video = new Hono(); @@ -12,43 +13,47 @@ video.get( zValidator("query", VideoSearchSchema), async (c) => { const params = c.req.valid("query"); - const page = params["start-index"] ?? 1; + const page = Number(params["start-index"] ?? 1); if (page > 5) { return c.html(search([], "")); } - // * fetch all videos up to current page const allVideos: YTNodes.Video[] = []; + let searchQuery = await innertube.search(params.q, { + type: "video", duration: params.duration, upload_date: params.time, sort_by: params.orderby }); - for (let i = 1; i <= page; i++) { + let currentPage = 1; + + while (currentPage <= page) { const videos = searchQuery.results.filter( (node): node is YTNodes.Video => node.type === "Video" ); + allVideos.push(...videos); - if (i < page) { - try { - const nextPage = await searchQuery.getContinuation(); - if (!nextPage) break; - searchQuery = nextPage; - } catch (err) { - // * no more pages available - break; - } - } + if (!searchQuery.has_continuation) break; + if (currentPage === page) break; + + searchQuery = await searchQuery.getContinuation(); + currentPage++; } // * build next page URL const nextUrl = new URL(c.req.url); - nextUrl.searchParams.set("start-index", (page + 1).toString()); + nextUrl.searchParams.set("start-index", String(page + 1)); - return c.html(search(allVideos, nextUrl.toString())); + return c.html( + search( + parseVideos(allVideos), + searchQuery.has_continuation ? nextUrl.toString() : "" + ) + ); } ); diff --git a/src/index.ts b/src/index.ts index e3da8f3..a04faa0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -8,6 +8,7 @@ import { config } from "./lib/config.js" import video from "./api/video.js" import playback from "./api/playback.js" import channel from "./api/channel.js" +import feed from "./api/feed.js" const app = new Hono() @@ -17,6 +18,7 @@ app.use("/schemas/*", serveStatic({ root: "./" })) app.route("/getvideo", playback); app.route("/feeds/api", video); app.route("/feeds/api", channel); +app.route("/feeds/api", feed); serve({ fetch: app.fetch, diff --git a/src/lib/channel.ts b/src/lib/channel.ts deleted file mode 100644 index a166521..0000000 --- a/src/lib/channel.ts +++ /dev/null @@ -1,16 +0,0 @@ -export function parseSubscriberCount(text: string) { - const cleaned = text.replace(/[^\d.KMB]/gi, ""); - const multipliers: Record = { - K: 1_000, - M: 1_000_000, - B: 1_000_000_000, - }; - - for (const [suffix, multiplier] of Object.entries(multipliers)) { - if (cleaned.toUpperCase().includes(suffix)) { - return Math.floor(parseFloat(cleaned) * multiplier); - } - } - - return parseInt(cleaned) || 0; -} \ No newline at end of file diff --git a/src/lib/config.ts b/src/lib/config.ts index 730a478..8929595 100644 --- a/src/lib/config.ts +++ b/src/lib/config.ts @@ -1,12 +1,25 @@ import { z } from "zod"; -import "dotenv/config" +import fs from "fs"; +import path from "path"; + +const cookiesPath = path.resolve(process.cwd(), "cookies.txt"); + +let cookies: string | null = null; + +if (fs.existsSync(cookiesPath)) { + cookies = fs.readFileSync(cookiesPath, "utf8").trim(); +} const EnvSchema = z.object({ - BASE_URL: z.url(), - PORT: z.coerce.number().default(4000) + BASE_URL: z.string().url(), + PORT: z.coerce.number().default(4000), + COOKIES: z.string().nullable().default(null), }); -const parsed = EnvSchema.safeParse(process.env); +const parsed = EnvSchema.safeParse({ + ...process.env, + COOKIES: cookies ?? null, +}); if (!parsed.success) { console.error("Invalid config", parsed.error.format()); diff --git a/src/lib/innertube.ts b/src/lib/innertube.ts index dba7c19..881820e 100644 --- a/src/lib/innertube.ts +++ b/src/lib/innertube.ts @@ -1,4 +1,5 @@ -import { Innertube, Platform, type Types } from 'youtubei.js'; +import { ClientType, Innertube, Platform, type Types, UniversalCache } from 'youtubei.js'; +import { config } from './config.js'; Platform.shim.eval = async ( data: Types.BuildScriptResult, @@ -12,5 +13,11 @@ Platform.shim.eval = async ( return new Function(code)(); }; -const innertube = await Innertube.create(); +const innertube = await Innertube.create({ + cache: new UniversalCache(true, "./cache"), + device_category: "mobile", + enable_session_cache: true, + client_type: ClientType.WEB, + ...(config.COOKIES ? { cookie: config.COOKIES } : {}) +}); export default innertube \ No newline at end of file diff --git a/src/lib/utils.ts b/src/lib/utils.ts new file mode 100644 index 0000000..c96e894 --- /dev/null +++ b/src/lib/utils.ts @@ -0,0 +1,66 @@ +export function parseCount(text: string) { + const cleaned = text.replace(/[^\d.KMB]/gi, ""); + const multipliers: Record = { + K: 1_000, + M: 1_000_000, + B: 1_000_000_000, + }; + + for (const [suffix, multiplier] of Object.entries(multipliers)) { + if (cleaned.toUpperCase().includes(suffix)) { + return Math.floor(parseFloat(cleaned) * multiplier); + } + } + + return parseInt(cleaned) || 0; +} + +export function parseDuration(duration: string) { + const parts = duration.split(":").map(Number); + + if (parts.length === 3) { + return parts[0] * 3600 + parts[1] * 60 + parts[2]; + } else if (parts.length === 2) { + return parts[0] * 60 + parts[1]; + } else if (parts.length === 1) { + return parts[0]; + } + + return 0; +} + +export function parseRelativeDate(text: string): string | null { + const now = new Date(); + const match = text.match(/(\d+)\s+(second|minute|hour|day|week|month|year)s?\s+ago/); + if (!match) return null; + + const unit = match[2]; + const date = new Date(now); + const value = parseInt(match[1]); + + switch (unit) { + case "second": + date.setSeconds(date.getSeconds() - value); + break; + case "minute": + date.setMinutes(date.getMinutes() - value); + break; + case "hour": + date.setHours(date.getHours() - value); + break; + case "day": + date.setDate(date.getDate() - value); + break; + case "week": + date.setDate(date.getDate() - value * 7); + break; + case "month": + date.setMonth(date.getMonth() - value); + break; + case "year": + date.setFullYear(date.getFullYear() - value); + break; + } + + return date.toISOString(); +} \ No newline at end of file diff --git a/src/lib/utils/channel.ts b/src/lib/utils/channel.ts new file mode 100644 index 0000000..fe309bf --- /dev/null +++ b/src/lib/utils/channel.ts @@ -0,0 +1,42 @@ +import { YTNodes, YT } from "youtubei.js"; +import { parseCount } from "../utils.js"; + +export interface Channel { + id: string; + name: string; + description: string; + thumbnail: string; + subCount: number; +} + +// TODO: make this not ugly +export function parseChannel(data: YT.Channel): Channel | null { + const { header, metadata } = data; + if (!header) return null; + + if (header.is(YTNodes.PageHeader)) { + return { + id: metadata?.external_id || "", + name: header.page_title, + description: header.content?.description?.description?.text || "", + thumbnail: (header.content?.image as YTNodes.DecoratedAvatarView) + ?.avatar?.image?.[0]?.url || "", + subCount: parseCount( + header.content?.metadata?.metadata_rows?.[1] + ?.metadata_parts?.[0]?.text?.text || "0" + ), + }; + } + + if (header.is(YTNodes.C4TabbedHeader)) { + return { + id: metadata?.external_id || "", + name: header.author?.name || "", + description: data.metadata?.description || "", + thumbnail: header.author?.best_thumbnail?.url || "", + subCount: parseCount(header.subscribers?.text || "0"), + }; + } + + return null; +} \ No newline at end of file diff --git a/src/lib/utils/feed.ts b/src/lib/utils/feed.ts new file mode 100644 index 0000000..93fbb98 --- /dev/null +++ b/src/lib/utils/feed.ts @@ -0,0 +1,74 @@ +import { YTNodes } from "youtubei.js"; +import { parseCount, parseDuration, parseRelativeDate } from "../utils.js"; +import { type Video } from "./video.js"; +import { YT } from "youtubei.js"; + +export function parseFeedVideo( + video: YTNodes.Video | YTNodes.RichItem +): Video | null { + const content = video.is(YTNodes.RichItem) ? video.content : video; + if (!content || content.type !== "LockupView") return null; + + const lockup = content as YTNodes.LockupView; + if (lockup.content_type !== "VIDEO") return null; + + const thumbnail = lockup.content_image as YTNodes.ThumbnailView; + const metadataRows = lockup.metadata?.metadata?.metadata_rows ?? []; + + // * extract author info + const authorPart = metadataRows[0]?.metadata_parts?.[0]?.text; + const authorName = authorPart?.text ?? ""; + const authorId = authorPart?.endpoint?.payload?.browseId ?? ""; + + // * extract view count and published date + const viewText = metadataRows[1]?.metadata_parts?.[0]?.text?.text ?? "0"; + const createdText = metadataRows[1]?.metadata_parts?.[1]?.text?.text ?? "0 seconds ago"; + + // * extract video duration + const overlay = thumbnail?.overlays[0] as YTNodes.ThumbnailOverlayBadgeView; + const durationText = overlay ? overlay.badges[0]?.text : undefined; + + return { + id: lockup.content_id ?? "", + title: lockup.metadata?.title?.text ?? "", + description: "", + thumbnailUrl: thumbnail?.image?.[0]?.url ?? "", + authorName, + authorId, + viewCount: parseCount(viewText), + created: parseRelativeDate(createdText) ?? "", + duration: parseDuration(durationText ?? "0:00"), + }; +} + +export async function parseHomeFeed(feed: YT.HomeFeed): Promise { + const videos: Video[] = []; + let currentFeed = feed; + + while (currentFeed.has_continuation) { + const grid = currentFeed.contents; + if (!grid || !grid.is(YTNodes.RichGrid)) break; + + for (const item of grid.contents) { + if (item.is(YTNodes.RichItem)) { + const parsed = parseFeedVideo(item); + if (parsed) videos.push(parsed); + } else if (item.is(YTNodes.RichSection)) { + const sectionContent = item.content; + if (sectionContent?.is(YTNodes.RichShelf)) { + for (const shelfItem of sectionContent.contents || []) { + if (shelfItem.is(YTNodes.RichItem)) { + const parsed = parseFeedVideo(shelfItem); + if (parsed) videos.push(parsed); + } + } + } + } + } + + const continuation = await currentFeed.getContinuation(); + currentFeed = continuation; + } + + return videos; +} \ No newline at end of file diff --git a/src/lib/utils/video.ts b/src/lib/utils/video.ts new file mode 100644 index 0000000..2b71e94 --- /dev/null +++ b/src/lib/utils/video.ts @@ -0,0 +1,40 @@ +import { YTNodes } from "youtubei.js"; +import { parseCount, parseDuration, parseRelativeDate } from "../utils.js"; + +export interface Video { + id: string; + title: string; + description: string; + authorId: string; + authorName: string; + duration: number; + viewCount: number; + thumbnailUrl: string; + created: string; +} + +export function parseVideos(videos: YTNodes.Video[]): Video[] { + return videos.map(video => parseVideo(video)); +} + +export function parseVideo(video: YTNodes.Video): Video { + const authorId = video.author?.id && video.author.id !== "N/A" + ? video.author.id + : video.video_id; + + const authorName = video.author?.name && video.author.name !== "N/A" + ? video.author.name + : "Unknown"; + + return { + id: video.video_id, + title: video.title?.text || "", + description: video.description || "", + authorId, + authorName, + duration: parseDuration(video.duration?.text || "0:00"), + viewCount: parseCount(video.view_count?.text || "0"), + thumbnailUrl: video.thumbnails?.[0]?.url || `https://i.ytimg.com/vi/${video.video_id}/default.jpg`, + created: parseRelativeDate(video?.published?.text || "0 seconds ago") ?? "" + }; +} \ No newline at end of file diff --git a/src/models/FeedGetSchema.ts b/src/models/FeedGetSchema.ts new file mode 100644 index 0000000..b098bf0 --- /dev/null +++ b/src/models/FeedGetSchema.ts @@ -0,0 +1,6 @@ +import z from "zod"; + +export const FeedGetSchema = z.object({ + region: z.string().length(2), + type: z.enum(["most_popular", "most_popular_Film", "most_popular_Games", "most_popular_Music"]) +}); \ No newline at end of file diff --git a/src/templates/channelData.ts b/src/templates/channelData.ts index aafed92..297a6fd 100644 --- a/src/templates/channelData.ts +++ b/src/templates/channelData.ts @@ -1,29 +1,26 @@ import { html } from "hono/html"; +import type { Channel } from "../lib/utils/channel.js"; export default ( - id: string, - name: string, - subCount: number, - description: string, - pictureUrl: string + channel: Channel ) => html` - tag:youtube.com,2008:channel:${id} + tag:youtube.com,2008:channel:${channel.id} 2014-09-16T18:07:05.000Z - ${name} - ${description} + ${channel.name} + ${channel.description} - - + + - ${name} + ${channel.name} /feeds/api/users/webauditors - ${id} + ${channel.id} - ${id} - + ${channel.id} + - + ` \ No newline at end of file diff --git a/src/templates/homeFeed.ts b/src/templates/homeFeed.ts new file mode 100644 index 0000000..88bbfae --- /dev/null +++ b/src/templates/homeFeed.ts @@ -0,0 +1,106 @@ +import { html } from "hono/html"; +import { config } from "../lib/config.js"; +import type { Video } from "../lib/utils/video.js"; + +export default ( + videos: Video[] = [] +) => html ` + + + tag:youtube.com,2008:playlist:8E2186857EE27746 + 2012-08-23T12:33:58.000Z + + + + http://www.gstatic.com/youtube/img/logo.png + + + + + + + + + + + YouTube data API + 50 + 50 + 1 + + ${videos.map(video => html` + + tag:youtube.com,2008:playlist:${video.id}:${video.id} + ${video.created} + ${video.created} + + + ${video.title} + + + + + + + + + + ${video.authorName} + ${config.BASE_URL}/ + {${video.authorId} + + + + + + + + + + + + + + + + + + Howto + + + + {${video.authorId} + ${video.description} + + youtube + + + + + + + + + + + + ${video.description}} + + ${video.created} + ${video.authorId} + ${video.id} + + + 1970-08-22 + + + 1 + + `)} + +` \ No newline at end of file diff --git a/src/templates/search.ts b/src/templates/search.ts index a84c350..2b66db2 100644 --- a/src/templates/search.ts +++ b/src/templates/search.ts @@ -1,62 +1,62 @@ import { html } from "hono/html"; import { config } from "../lib/config.js"; -import { YTNodes } from "youtubei.js"; +import type { Video } from "../lib/utils/video.js"; export default ( - videos: YTNodes.Video[], + videos: Video[], nextPage: string ) => html` + xmlns="http://www.w3.org/2005/Atom" + xmlns:gd="http://schemas.google.com/g/2005" + xmlns:openSearch="http://a9.com/-/spec/opensearch/1.1/" + xmlns:yt="${config.BASE_URL}/schemas/2007" + xmlns:media="http://search.yahoo.com/mrss/" + gd:etag="W/"DkcBQ3g-fip7I2A9XRRXEUw.""> - tag:youtube.com,2008:channels - ${new Date().toISOString()} - - Channels matching: webauditors - http://www.gstatic.com/youtube/img/logo.png + tag:youtube.com,2008:channels + ${new Date().toISOString()} + + Channels matching: webauditors + http://www.gstatic.com/youtube/img/logo.png - - - - - - ${nextPage ? html`` : ""} + + + + + + ${nextPage ? html`` : ""} - - YouTube - http://www.youtube.com/ - + + YouTube + http://www.youtube.com/ + - YouTube data API - ${videos.length} - 1 - ${videos.length} + YouTube data API + ${videos.length} + 1 + ${videos.length} - ${videos.map(video => html` + ${videos.map(video => html` - tag:youtube.com,2008:playlist:${video.video_id}:${video.video_id} - ${new Date().toISOString()} - ${new Date().toISOString()} + tag:youtube.com,2008:playlist:${video.id}:${video.id} + ${video.created} + ${video.created} ${video.title} - - - - - - + + + + + + - ${video.author.name} - https://gdata.youtube.com/feeds/api/users/${video.author.id} - ${video.author.id} + ${video.authorName} + https://gdata.youtube.com/feeds/api/users/${video.authorId} + ${video.authorId} @@ -67,38 +67,38 @@ export default ( - + Paris ,FR Howto - - - - ${video.author.id} + + + + ${video.authorId} ${video.description} youtube - - - - - - - - - - - + + + + + + + + + + + ${video.title} - - ${new Date().toISOString()} - ${video.author.id} - ${video.video_id} + + ${video.created} + ${video.authorId} + ${video.id} 1970-08-22 - + 1