fix: missing referer header

.gitignore

@@ -2,3 +2,4 @@ node_modules/
 external/
 dist/
 nogit/
+ralph-loop.local.md

src/detection/dom-monitor.ts

@@ -6,16 +6,32 @@ function isM3U8Url(url: string): boolean {
          lower.includes('application/x-mpegurl') || lower.includes('application/vnd.apple.mpegurl');
 }
 
-export function monitorDOM(onDetected: (url: string) => void): void {
+export function monitorDOM(onDetected: (url: string, headers?: Record<string, string>) => void): void {
   const target = (typeof unsafeWindow !== 'undefined' ? unsafeWindow : window) as any;
   const doc = target.document || document;
   if (!doc) return;
 
+  let ctxReferer = '';
+  try {
+    ctxReferer = location.href;
+  } catch { /* cross-origin */ }
+
+  const getReferer = (src: string): string => {
+    try {
+      const srcObj = new URL(src);
+      if (srcObj.origin !== location.origin) {
+        return location.origin + '/';
+      }
+    } catch { /* ignore */ }
+    return ctxReferer;
+  };
+
   const checkSource = (el: HTMLMediaElement) => {
     const src = el.currentSrc || el.src || '';
     if (src && isM3U8Url(src)) {
       log.info(`DOM monitor: found m3u8 in <${el.tagName.toLowerCase()}> src: ${src}`);
-      onDetected(src);
+      const referer = getReferer(src);
+      onDetected(src, referer ? { Referer: referer } : undefined);
     }
 
     if (el.children) {
@@ -24,7 +40,8 @@ export function monitorDOM(onDetected: (url: string) => void): void {
         const sourceSrc = source.src || '';
         if (sourceSrc && isM3U8Url(sourceSrc)) {
           log.info(`DOM monitor: found m3u8 in <source> src: ${sourceSrc}`);
-          onDetected(sourceSrc);
+          const referer = getReferer(sourceSrc);
+          onDetected(sourceSrc, referer ? { Referer: referer } : undefined);
         }
       }
     }
@@ -49,7 +66,8 @@ export function monitorDOM(onDetected: (url: string) => void): void {
             const src = el.getAttribute('src') || '';
             if (isM3U8Url(src)) {
               log.info(`DOM monitor: found m3u8 in <source> tag: ${src}`);
-              onDetected(src);
+              const referer = getReferer(src);
+              onDetected(src, referer ? { Referer: referer } : undefined);
             }
           }
         }

src/detection/fetch-intercept.ts

@@ -7,7 +7,7 @@ function isM3U8Url(url: string): boolean {
   return M3U8_PATTERNS.some(p => lower.includes(p));
 }
 
-export function interceptFetch(onDetected: (url: string) => void): void {
+export function interceptFetch(onDetected: (url: string, headers?: Record<string, string>) => void): void {
   const target = (typeof unsafeWindow !== 'undefined' ? unsafeWindow : window) as any;
   const origFetch = target.fetch;
   if (!origFetch) return;
@@ -27,7 +27,14 @@ export function interceptFetch(onDetected: (url: string) => void): void {
         if (response.status >= 200 && response.status < 300) {
           const isM3U8Response = isM3U8Url(url) || contentType.toLowerCase().includes('mpegurl');
           if (isM3U8Response) {
-            onDetected(url);
+            try {
+              let referer = location.href;
+              const urlObj = new URL(url);
+              if (urlObj.origin !== location.origin) {
+                referer = location.origin + '/';
+              }
+              onDetected(url, { Referer: referer });
+            } catch { /* cross-origin */ }
           }
         }
       }

src/detection/xhr-intercept.ts

@@ -42,12 +42,21 @@ export function interceptXHR(onDetected: (url: string, headers?: Record<string,
             if (status >= 200 && status < 300) {
               const isM3U8Response = isM3U8Url(requestUrl) || contentType.toLowerCase().includes('mpegurl') || responseText.startsWith('#EXTM3U');
               if (isM3U8Response) {
-                const capturedHeaders = Object.keys(headers).length ? headers : {};
+                const capturedHeaders: Record<string, string> = {};
-                if (!capturedHeaders.Referer) {
+                for (const [k, v] of Object.entries(headers)) {
-                  try {
+                  if (k.toLowerCase() !== 'referer') {
-                    capturedHeaders.Referer = location.href;
+                    capturedHeaders[k] = v;
-                  } catch { /* cross-origin */ }
                   }
+                }
+                try {
+                  const requestUrlObj = new URL(requestUrl);
+                  const currentOrigin = location.origin;
+                  if (requestUrlObj.origin !== currentOrigin) {
+                    capturedHeaders.Referer = currentOrigin + '/';
+                  } else {
+                    capturedHeaders.Referer = location.href;
+                  }
+                } catch { /* cross-origin */ }
                 onDetected(requestUrl, capturedHeaders);
               }
             }

src/downloader/segment-fetch.ts

@@ -1,4 +1,5 @@
 import { log } from '../logger';
+import { normalizeReferer } from '../utils/uri';
 
 export interface DownloadProgress {
   current: number;
@@ -18,14 +19,17 @@ function gmFetchBuffer(url: string, referer?: string, origin?: string): Promise<
   const fallbackOrigin = new URL(document.location.href).origin;
   const effectiveReferer = referer || fallbackReferer;
   const effectiveOrigin = origin || (referer ? new URL(referer).origin : fallbackOrigin);
+  const normalizedReferer = normalizeReferer(url, effectiveReferer);
+
+  log.info(`gmFetchBuffer request: url=${url} referer=${normalizedReferer} origin=${effectiveOrigin}`);
 
   return new Promise((resolve, reject) => {
     (GM_xmlhttpRequest as any)({
       method: 'GET',
       url: url,
       responseType: 'arraybuffer',
-      referer: effectiveReferer,
       headers: {
+        'Referer': normalizedReferer,
         'Origin': effectiveOrigin,
       },
       onload: (response: any) => {
@@ -36,10 +40,12 @@ function gmFetchBuffer(url: string, referer?: string, origin?: string): Promise<
             bytes: buffer.byteLength,
           });
         } else {
+          log.error(`gmFetchBuffer HTTP error: status=${response.status} url=${url} referer=${effectiveReferer} origin=${effectiveOrigin} headers=${JSON.stringify(response.responseHeaders)}`);
           reject(new Error(`Segment fetch failed: HTTP ${response.status} for ${url}`));
         }
       },
       onerror: (error: any) => {
+        log.error(`gmFetchBuffer network error: url=${url} referer=${effectiveReferer} origin=${effectiveOrigin} error=${error.error} status=${error.status} readyState=${error.readyState} responseHeaders=${JSON.stringify(error.responseHeaders || {})}`);
         reject(new Error(`Segment fetch error: ${error.error} for ${url}`));
       },
     });

src/index.ts

@@ -1,4 +1,5 @@
 import { log } from './logger';
+import { normalizeReferer } from './utils/uri';
 import { interceptXHR } from './detection/xhr-intercept';
 import { interceptFetch } from './detection/fetch-intercept';
 import { monitorDOM } from './detection/dom-monitor';
@@ -26,13 +27,15 @@ function isTopFrame(): boolean {
 function notifyTopFrame(url: string, headers?: Record<string, string>): void {
   if (!isTopFrame()) {
     try {
+      const referer = headers?.Referer || location.href;
+      const normalizedReferer = normalizeReferer(url, referer);
       window.top.postMessage({
         __m3u8dl: true,
         url,
-        referer: headers?.Referer || location.href,
+        referer: normalizedReferer,
         origin: location.origin,
       }, '*');
-      log.info(`notifyTopFrame: sent via postMessage: ${url}`);
+      log.info(`notifyTopFrame: sent via postMessage: ${url} referer=${normalizedReferer}`);
     } catch (e) {
       log.warn(`notifyTopFrame: postMessage failed: ${e}`);
     }
@@ -85,6 +88,7 @@ async function startDownload(url: string): Promise<void> {
   downloadAborted = false;
 
   const entry = detectedM3U8s.get(url);
+  log.info(`startDownload: ${url} entryHeaders=${JSON.stringify(entry?.headers)}`);
   let parsed = entry?.parsed;
 
   if (!parsed) {
@@ -123,7 +127,7 @@ async function startDownload(url: string): Promise<void> {
   }
 
   const segmentUrls = parsed.segments.map(s => s.uri);
-  const referrer = entry?.headers?.Referer;
+  const referrer = entry?.headers?.Referer || document.location.href;
   const requestOrigin = entry?.headers?.Origin;
 
   resetProgress();
@@ -159,7 +163,7 @@ async function startDownload(url: string): Promise<void> {
 function onM3U8Detected(url: string, headers?: Record<string, string>): void {
   if (detectedM3U8s.has(url)) return;
 
-  log.info(`onM3U8Detected: ${url}`);
+  log.info(`onM3U8Detected: ${url} headers=${JSON.stringify(headers)}`);
   detectedM3U8s.set(url, { url, headers });
 
   notifyTopFrame(url, headers);

src/parser/key-fetch.ts

@@ -1,23 +1,25 @@
 import { log } from '../logger';
+import { normalizeReferer } from '../utils/uri';
 
 export async function fetchKey(keyUri: string, referer?: string, origin?: string): Promise<Uint8Array> {
-  log.info(`fetchKey: ${keyUri}`);
-
   const fallbackReferer = document.location.href;
   const fallbackOrigin = new URL(document.location.href).origin;
-  const effectiveReferer = referer || fallbackReferer;
+  const effectiveReferer = normalizeReferer(keyUri, referer || fallbackReferer);
   const effectiveOrigin = origin || (referer ? new URL(referer).origin : fallbackOrigin);
 
+  log.info(`fetchKey request: url=${keyUri} referer=${effectiveReferer} origin=${effectiveOrigin}`);
+
   return new Promise((resolve, reject) => {
     (GM_xmlhttpRequest as any)({
       method: 'GET',
       url: keyUri,
       responseType: 'arraybuffer',
-      referer: effectiveReferer,
       headers: {
+        'Referer': effectiveReferer,
         'Origin': effectiveOrigin,
       },
       onload: (response: any) => {
+        log.info(`fetchKey response: status=${response.status} readyState=${response.readyState} headers=${JSON.stringify(response.responseHeaders)}`);
         if (response.status >= 200 && response.status < 300) {
           const buffer = response.detail?.response || response.response;
           const keyBytes = new Uint8Array(buffer);
@@ -29,10 +31,12 @@ export async function fetchKey(keyUri: string, referer?: string, origin?: string
 
           resolve(keyBytes);
         } else {
+          log.error(`fetchKey HTTP error: status=${response.status} url=${keyUri} responseHeaders=${JSON.stringify(response.responseHeaders)}`);
           reject(new Error(`Key fetch failed: HTTP ${response.status}`));
         }
       },
       onerror: (error: any) => {
+        log.error(`fetchKey network error: url=${keyUri} error=${error.error} status=${error.status} readyState=${error.readyState} responseHeaders=${JSON.stringify(error.responseHeaders || {})}`);
         reject(new Error(`Key fetch error: ${error.error}`));
       },
     });

src/parser/m3u8-parse.ts

@@ -1,6 +1,6 @@
 import * as m3u8Parser from 'm3u8-parser';
 import { log } from '../logger';
-import { resolveUri } from '../utils/uri';
+import { resolveUri, normalizeReferer } from '../utils/uri';
 import { fetchKey } from './key-fetch';
 
 export interface SegmentInfo {
@@ -34,30 +34,42 @@ function gmFetchText(url: string, referer?: string, origin?: string): Promise<st
   const effectiveReferer = referer || fallbackReferer;
   const effectiveOrigin = origin || (referer ? new URL(referer).origin : fallbackOrigin);
 
+  const normalizedReferer = normalizeReferer(url, effectiveReferer);
+
+  log.info(`gmFetchText request: url=${url} referer=${normalizedReferer} origin=${effectiveOrigin}`);
+
   return new Promise((resolve, reject) => {
     (GM_xmlhttpRequest as any)({
       method: 'GET',
       url: url,
       responseType: 'text',
-      referer: effectiveReferer,
       headers: {
+        'Referer': normalizedReferer,
         'Origin': effectiveOrigin,
       },
       onload: (response: any) => {
+        log.info(`gmFetchText response: status=${response.status} readyState=${response.readyState} headers=${JSON.stringify(response.responseHeaders)}`);
         if (response.status >= 200 && response.status < 300) {
           resolve(response.responseText || response.response || '');
         } else {
+          log.error(`gmFetchText HTTP error: status=${response.status} url=${url} responseHeaders=${JSON.stringify(response.responseHeaders)} bodyLen=${(response.responseText || '').length} bodyPreview=${(response.responseText || '').substring(0, 200)}`);
           reject(new Error(`Failed to fetch ${url}: HTTP ${response.status}`));
         }
       },
       onerror: (error: any) => {
+        log.error(`gmFetchText network error: url=${url} error=${error.error} status=${error.status} readyState=${error.readyState} responseHeaders=${JSON.stringify(error.responseHeaders || {})}`);
         reject(new Error(`Failed to fetch ${url}: ${error.error}`));
       },
     });
   });
 }
 
-export async function parseM3U8(url: string, referer?: string, origin?: string): Promise<ParsedPlaylist> {
+export async function parseM3U8(
+  url: string,
+  referer?: string,
+  origin?: string,
+  fetchKeys = true
+): Promise<ParsedPlaylist> {
   log.info(`parseM3U8: ${url}`);
 
   const content = await gmFetchText(url, referer, origin);
@@ -108,7 +120,9 @@ export async function parseM3U8(url: string, referer?: string, origin?: string):
         log.warn(`parseM3U8: unsupported encryption method ${method}`);
       }
 
-      let keyBytes = keyCache.get(keyUri);
+      let keyBytes: Uint8Array | null = null;
+      if (fetchKeys) {
+        keyBytes = keyCache.get(keyUri);
         if (!keyBytes) {
           try {
             keyBytes = await fetchKey(keyUri, referer, origin);
@@ -117,6 +131,7 @@ export async function parseM3U8(url: string, referer?: string, origin?: string):
             log.error(`parseM3U8: failed to fetch key ${keyUri}: ${e}`);
           }
         }
+      }
 
       let iv: Uint8Array | null = null;
       if (segment.key.iv) {
@@ -140,7 +155,7 @@ export async function parseM3U8(url: string, referer?: string, origin?: string):
     resolvedSegments.push({ uri, key: keyInfo });
   }
 
-  log.info(`parseM3U8: ${resolvedSegments.length} segments, encrypted=${resolvedSegments.some(s => s.key !== null)}`);
+  log.info(`parseM3U8: ${resolvedSegments.length} segments, encrypted=${resolvedSegments.some(s => s.key !== null)}, fetchKeys=${fetchKeys}`);
   return {
     segments: resolvedSegments,
     isMaster: false,

src/utils/uri.ts

@@ -1,3 +1,14 @@
+export function normalizeReferer(targetUrl: string, referer: string): string {
+  try {
+    const targetOrigin = new URL(targetUrl).origin;
+    const refererOrigin = new URL(referer).origin;
+    if (targetOrigin !== refererOrigin) {
+      return refererOrigin + '/';
+    }
+  } catch { /* ignore */ }
+  return referer;
+}
+
 export function resolveUri(root: string, rel: string): string {
   if (rel.startsWith('http://') || rel.startsWith('https://') || rel.startsWith('//')) {
     return rel;