Initial commit
This commit is contained in:
+355
@@ -0,0 +1,355 @@
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { execFile } from 'node:child_process';
|
||||
import { info, warn } from './logger.mjs';
|
||||
|
||||
const DEFAULT_BIN = process.env.CALIBRE_FETCH_BIN || 'fetch-ebook-metadata';
|
||||
const DEFAULT_DEBUG_BIN = process.env.CALIBRE_DEBUG_BIN || 'calibre-debug';
|
||||
const DEFAULT_TIMEOUT_MS = 75_000;
|
||||
const MAX_LOG_CHARS = 4000;
|
||||
|
||||
// Machine-readable identification pass. Calibre's fetch-ebook-metadata CLI
|
||||
// prints useful bibliographic metadata, but its human-readable/verbose output
|
||||
// is not stable enough to parse. This fixed calibre-debug command asks
|
||||
// Calibre's own identify pipeline for the best matches and emits only a JSON
|
||||
// marker that kovi can consume. User-controlled metadata is supplied via
|
||||
// environment variables and is never interpolated into Python source.
|
||||
const IDENTIFY_JSON_CODE = [
|
||||
'import os, json',
|
||||
'from io import BytesIO',
|
||||
'from threading import Event',
|
||||
'from calibre.ebooks.metadata import string_to_authors',
|
||||
'from calibre.ebooks.metadata.sources.base import create_log',
|
||||
'from calibre.ebooks.metadata.sources.identify import identify',
|
||||
'from calibre.ebooks.metadata.sources.update import patch_plugins',
|
||||
'patch_plugins()',
|
||||
"title=os.environ.get('KOVI_CALIBRE_TITLE') or None",
|
||||
"author=os.environ.get('KOVI_CALIBRE_AUTHORS') or ''",
|
||||
'authors=string_to_authors(author) if author else []',
|
||||
'identifiers={}',
|
||||
"isbn=os.environ.get('KOVI_CALIBRE_ISBN') or ''",
|
||||
"identifiers.update({'isbn': isbn}) if isbn else None",
|
||||
'buf=BytesIO(); log=create_log(buf)',
|
||||
"results=identify(log,Event(),title=title,authors=authors,identifiers=identifiers,timeout=int(os.environ.get('KOVI_CALIBRE_TIMEOUT','45')))",
|
||||
"payload=[{'title':getattr(m,'title','') or '', 'authors':list(getattr(m,'authors',[]) or []), 'identifiers':m.get_identifiers() if hasattr(m,'get_identifiers') else {}, 'languages':list(getattr(m,'languages',[]) or [])} for m in results[:5]]",
|
||||
"print('KOVI_METADATA_JSON='+json.dumps(payload,ensure_ascii=False,separators=(',',':')))",
|
||||
].join(';');
|
||||
|
||||
// Calibre's fetch-ebook-metadata CLI only reaches the cover phase after the
|
||||
// identify phase succeeds. Google Images is a cover-only source and is disabled
|
||||
// by default in Calibre, so hard cases with translated titles can otherwise
|
||||
// never reach it. This fixed command runs Calibre's own cover downloader in a
|
||||
// separate calibre-debug process, explicitly enabling Google Images. Metadata
|
||||
// enters only through environment variables, never interpolated Python/source.
|
||||
const DIRECT_COVER_CODE = [
|
||||
'import os',
|
||||
'from io import BytesIO',
|
||||
'from calibre.customize.ui import enable_plugin',
|
||||
"enable_plugin('Google Images')",
|
||||
'from calibre.ebooks.metadata import string_to_authors',
|
||||
'from calibre.ebooks.metadata.sources.base import create_log',
|
||||
'from calibre.ebooks.metadata.sources.covers import download_cover',
|
||||
"title=os.environ.get('KOVI_CALIBRE_TITLE') or None",
|
||||
"author=os.environ.get('KOVI_CALIBRE_AUTHORS') or ''",
|
||||
'authors=string_to_authors(author) if author else []',
|
||||
'identifiers={}',
|
||||
"isbn=os.environ.get('KOVI_CALIBRE_ISBN') or ''",
|
||||
"identifiers.update({'isbn': isbn}) if isbn else None",
|
||||
'buf=BytesIO(); log=create_log(buf)',
|
||||
"cover=download_cover(log,title=title,authors=authors,identifiers=identifiers,timeout=int(os.environ.get('KOVI_CALIBRE_TIMEOUT','60')))",
|
||||
"out=os.environ['KOVI_CALIBRE_OUTPUT']",
|
||||
"open(out,'wb').write(cover[-1]) if cover else None",
|
||||
"print('provider='+cover[0].name if cover else 'no-cover')",
|
||||
"print(buf.getvalue().decode('utf-8','replace') if hasattr(buf.getvalue(),'decode') else str(buf.getvalue()))",
|
||||
].join(';');
|
||||
|
||||
function enabledByEnv() {
|
||||
const raw = String(process.env.CALIBRE_COVER_RESOLVER ?? 'auto').trim().toLowerCase();
|
||||
return !['0', 'false', 'no', 'off', 'disabled'].includes(raw);
|
||||
}
|
||||
|
||||
function cleanAuthor(value='') {
|
||||
const s = String(value || '').trim();
|
||||
if (!s || /^(n\/?a|unknown|sconosciuto)$/i.test(s)) return '';
|
||||
return s;
|
||||
}
|
||||
|
||||
function stripQtNoise(value='') {
|
||||
return String(value || '')
|
||||
.replace(/^.*(?:QRhiGles2|QVulkanInstance|WebEngineContext|Unable to detect GPU vendor|createPlatformVulkanInstance).*$/gmi, '')
|
||||
.replace(/\n{3,}/g, '\n\n');
|
||||
}
|
||||
|
||||
function trimLog(value='') {
|
||||
const s = stripQtNoise(value).replace(/\r/g, '').trim();
|
||||
if (!s) return '';
|
||||
return s.length > MAX_LOG_CHARS ? `${s.slice(0, MAX_LOG_CHARS)}…` : s;
|
||||
}
|
||||
|
||||
function parseMetadataJson(value='') {
|
||||
const text=String(value||'');
|
||||
const line=text.split(/\r?\n/).find(x=>x.startsWith('KOVI_METADATA_JSON='));
|
||||
if(!line) return [];
|
||||
try {
|
||||
const parsed=JSON.parse(line.slice('KOVI_METADATA_JSON='.length));
|
||||
if(!Array.isArray(parsed)) return [];
|
||||
return parsed.slice(0,5).map(x=>({
|
||||
title:String(x?.title||'').slice(0,1000),
|
||||
authors:Array.isArray(x?.authors)?x.authors.map(a=>String(a).slice(0,500)).slice(0,10):[],
|
||||
identifiers:x?.identifiers && typeof x.identifiers==='object' ? Object.fromEntries(Object.entries(x.identifiers).slice(0,20).map(([k,v])=>[String(k).slice(0,80),String(v).slice(0,500)])) : {},
|
||||
languages:Array.isArray(x?.languages)?x.languages.map(v=>String(v).slice(0,40)).slice(0,10):[],
|
||||
}));
|
||||
} catch { return []; }
|
||||
}
|
||||
|
||||
function execFilePromise(file, args, options, impl = execFile) {
|
||||
return new Promise((resolve, reject) => {
|
||||
impl(file, args, options, (error, stdout, stderr) => {
|
||||
if (error) {
|
||||
error.stdout = stdout;
|
||||
error.stderr = stderr;
|
||||
reject(error);
|
||||
} else resolve({ stdout, stderr });
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function normalizeTitle(value='') {
|
||||
return String(value).toLocaleLowerCase().normalize('NFKD').replace(/[\u0300-\u036f]/g,'').replace(/[^\p{L}\p{N}]+/gu,' ').trim();
|
||||
}
|
||||
|
||||
export function calibreTitleVariants(title='') {
|
||||
const raw=String(title||'').trim();
|
||||
const out=[];
|
||||
const push=(value)=>{
|
||||
const v=String(value||'').trim();
|
||||
if(v && !out.some(x=>normalizeTitle(x)===normalizeTitle(v))) out.push(v);
|
||||
};
|
||||
push(raw);
|
||||
push(raw.replace(/\s*[\[(].*?[\])]\s*$/,'').trim());
|
||||
const dot=raw.match(/^.{2,32}\.\s+(.{3,})$/); if(dot) push(dot[1]);
|
||||
const colon=raw.match(/^.{2,32}:\s+(.{3,})$/); if(colon) push(colon[1]);
|
||||
return out.slice(0,3);
|
||||
}
|
||||
|
||||
export function calibreArgsForBook(book, coverPath, {includeAuthor=true, includeIsbn=true, titleOverride=null}={}) {
|
||||
const args = [];
|
||||
const title=titleOverride ?? book.title;
|
||||
if (title) args.push('--title', String(title).trim());
|
||||
args.push('--cover', coverPath, '--timeout', '30');
|
||||
const author = cleanAuthor(book.authors);
|
||||
if (includeAuthor && author) args.push('--authors', author);
|
||||
if (includeIsbn && book.isbn) args.push('--isbn', String(book.isbn));
|
||||
args.push('--verbose');
|
||||
return args;
|
||||
}
|
||||
|
||||
export function calibreQueryPlans(book) {
|
||||
const plans=[];
|
||||
const push=(name,opts)=>{
|
||||
const key=JSON.stringify(opts);
|
||||
if(!plans.some(p=>JSON.stringify(p.opts)===key)) plans.push({name,opts});
|
||||
};
|
||||
if(book.isbn) push('isbn-title-author',{includeAuthor:true,includeIsbn:true,titleOverride:book.title});
|
||||
for(const [i,title] of calibreTitleVariants(book.title).entries()) {
|
||||
const label=i===0?'title':`title-variant-${i}`;
|
||||
push(`${label}-author`,{includeAuthor:true,includeIsbn:false,titleOverride:title});
|
||||
push(`${label}-only`,{includeAuthor:false,includeIsbn:false,titleOverride:title});
|
||||
}
|
||||
return plans.slice(0,5);
|
||||
}
|
||||
|
||||
function directCoverPlans(book) {
|
||||
const author=cleanAuthor(book.authors);
|
||||
const plans=[];
|
||||
for(const [i,title] of calibreTitleVariants(book.title).entries()) {
|
||||
const label=i===0?'direct-title':`direct-title-variant-${i}`;
|
||||
if(author) plans.push({name:`${label}-author`,title,author});
|
||||
plans.push({name:label,title,author:''});
|
||||
}
|
||||
return plans.slice(0,5);
|
||||
}
|
||||
|
||||
function calibreEnv(baseEnv, configDir, tmpDir, extra={}) {
|
||||
return {
|
||||
...baseEnv,
|
||||
CALIBRE_CONFIG_DIRECTORY: configDir,
|
||||
CALIBRE_TEMP_DIR: path.join(tmpDir,'tmp'),
|
||||
CALIBRE_CACHE_DIRECTORY: path.join(tmpDir,'cache'),
|
||||
QT_QPA_PLATFORM: baseEnv.QT_QPA_PLATFORM || 'offscreen',
|
||||
QT_OPENGL: baseEnv.QT_OPENGL || 'software',
|
||||
QT_QUICK_BACKEND: baseEnv.QT_QUICK_BACKEND || 'software',
|
||||
LIBGL_ALWAYS_SOFTWARE: baseEnv.LIBGL_ALWAYS_SOFTWARE || '1',
|
||||
QTWEBENGINE_DISABLE_SANDBOX: baseEnv.QTWEBENGINE_DISABLE_SANDBOX || '1',
|
||||
QTWEBENGINE_CHROMIUM_FLAGS: baseEnv.QTWEBENGINE_CHROMIUM_FLAGS || '--disable-gpu --disable-dev-shm-usage --no-sandbox',
|
||||
...extra,
|
||||
};
|
||||
}
|
||||
|
||||
async function runDirectCoverStage(book,{tmpDir,configDir,execFileImpl,debugBin,timeoutMs,env}) {
|
||||
const plans=directCoverPlans(book);
|
||||
info('cover.calibre.direct.start','Running Calibre cover-only sources, including Google Images.',{
|
||||
bookId:book.id,title:book.title,plans:plans.map(p=>p.name),
|
||||
});
|
||||
let lastDetail='';
|
||||
for(let i=0;i<plans.length;i++){
|
||||
const plan=plans[i];
|
||||
const output=path.join(tmpDir,`direct-${i}.jpg`);
|
||||
info('cover.calibre.direct.attempt','Running Calibre cover-only query.',{bookId:book.id,title:book.title,strategy:plan.name,attempt:i+1,queryTitle:plan.title});
|
||||
try{
|
||||
const result=await execFilePromise(debugBin,['--command',DIRECT_COVER_CODE],{
|
||||
timeout:timeoutMs,
|
||||
maxBuffer:2*1024*1024,
|
||||
windowsHide:true,
|
||||
env:calibreEnv(env,configDir,tmpDir,{
|
||||
KOVI_CALIBRE_TITLE:plan.title,
|
||||
KOVI_CALIBRE_AUTHORS:plan.author,
|
||||
KOVI_CALIBRE_ISBN:String(book.isbn||''),
|
||||
KOVI_CALIBRE_OUTPUT:output,
|
||||
KOVI_CALIBRE_TIMEOUT:'60',
|
||||
}),
|
||||
},execFileImpl);
|
||||
lastDetail=trimLog(`${result?.stdout||''}\n${result?.stderr||''}`);
|
||||
}catch(e){
|
||||
if(e?.code==='ENOENT') return {status:'unavailable',reason:'calibre-debug-not-installed'};
|
||||
lastDetail=trimLog(e?.stderr||e?.stdout||e?.message);
|
||||
}
|
||||
if(fs.existsSync(output)){
|
||||
const buffer=fs.readFileSync(output);
|
||||
info('cover.calibre.direct.result','Calibre cover-only sources returned a candidate.',{bookId:book.id,title:book.title,strategy:plan.name,bytes:buffer.length,detail:lastDetail||undefined});
|
||||
return {status:'matched',buffer,strategy:plan.name,detail:lastDetail};
|
||||
}
|
||||
info('cover.calibre.direct.none','Calibre cover-only query returned no cover.',{bookId:book.id,title:book.title,strategy:plan.name,detail:lastDetail||undefined});
|
||||
}
|
||||
return {status:'none',reason:'direct-cover-none',detail:lastDetail};
|
||||
}
|
||||
|
||||
async function runMetadataIdentifyStage(book,{tmpDir,configDir,execFileImpl,debugBin,timeoutMs,env}) {
|
||||
info('cover.calibre.metadata.start','Asking Calibre for machine-readable identifiers after cover download failed.',{
|
||||
bookId:book.id,title:book.title,authors:book.authors,isbn:book.isbn||undefined,
|
||||
});
|
||||
try {
|
||||
const result=await execFilePromise(debugBin,['--command',IDENTIFY_JSON_CODE],{
|
||||
timeout:Math.min(timeoutMs,60_000),
|
||||
maxBuffer:2*1024*1024,
|
||||
windowsHide:true,
|
||||
env:calibreEnv(env,configDir,tmpDir,{
|
||||
KOVI_CALIBRE_TITLE:String(book.title||''),
|
||||
KOVI_CALIBRE_AUTHORS:cleanAuthor(book.authors),
|
||||
KOVI_CALIBRE_ISBN:String(book.isbn||''),
|
||||
KOVI_CALIBRE_TIMEOUT:'45',
|
||||
}),
|
||||
},execFileImpl);
|
||||
const candidates=parseMetadataJson(`${result?.stdout||''}\n${result?.stderr||''}`);
|
||||
info(candidates.length?'cover.calibre.metadata.result':'cover.calibre.metadata.none',candidates.length?'Calibre identified bibliographic records that can be reused for deterministic cover URLs.':'Calibre did not identify reusable bibliographic records.',{
|
||||
bookId:book.id,title:book.title,candidates:candidates.length,
|
||||
identifiers:candidates.slice(0,3).map(x=>Object.keys(x.identifiers||{})),
|
||||
});
|
||||
return candidates;
|
||||
} catch(e) {
|
||||
if(e?.code==='ENOENT') return [];
|
||||
warn('cover.calibre.metadata.error','Calibre metadata identification failed after cover lookup.',{bookId:book.id,title:book.title,error:trimLog(e?.stderr||e?.stdout||e?.message)});
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ask Calibre's own metadata-source framework to find the best cover. First
|
||||
* use fetch-ebook-metadata (Calibre's identify -> cover path). If identify
|
||||
* cannot find the book, run Calibre's cover downloader directly so cover-only
|
||||
* sources such as Google Images still get a chance.
|
||||
*/
|
||||
export async function fetchCoverWithCalibre(book, {
|
||||
dataDir,
|
||||
execFileImpl = execFile,
|
||||
bin = DEFAULT_BIN,
|
||||
debugBin = DEFAULT_DEBUG_BIN,
|
||||
timeoutMs = DEFAULT_TIMEOUT_MS,
|
||||
env = process.env,
|
||||
} = {}) {
|
||||
if (!enabledByEnv()) return { status:'unavailable', reason:'disabled' };
|
||||
if (!book?.title && !book?.isbn) return { status:'none', reason:'insufficient-metadata' };
|
||||
|
||||
const base = path.join(dataDir || os.tmpdir(), 'uploads');
|
||||
fs.mkdirSync(base, { recursive:true });
|
||||
const tmpDir = fs.mkdtempSync(path.join(base, 'calibre-cover-'));
|
||||
const configDir = path.join(tmpDir, 'config');
|
||||
fs.mkdirSync(configDir, { recursive:true });
|
||||
fs.mkdirSync(path.join(tmpDir,'tmp'),{recursive:true});
|
||||
fs.mkdirSync(path.join(tmpDir,'cache'),{recursive:true});
|
||||
const started = Date.now();
|
||||
const plans = calibreQueryPlans(book);
|
||||
|
||||
info('cover.calibre.start', 'Asking Calibre metadata sources for a cover.', {
|
||||
bookId:book.id, title:book.title, authors:book.authors, isbn:book.isbn || undefined, plans:plans.map(p=>p.name),
|
||||
});
|
||||
|
||||
try {
|
||||
let lastDetail='';
|
||||
for (let i=0;i<plans.length;i++) {
|
||||
const plan=plans[i];
|
||||
const coverPath = path.join(tmpDir, `cover-${i}.jpg`);
|
||||
const args = calibreArgsForBook(book, coverPath, plan.opts);
|
||||
info('cover.calibre.attempt','Running a Calibre identify/cover query.',{bookId:book.id,title:book.title,strategy:plan.name,attempt:i+1,queryTitle:plan.opts.titleOverride||book.title});
|
||||
let result;
|
||||
try {
|
||||
result = await execFilePromise(bin, args, {
|
||||
timeout: timeoutMs,
|
||||
maxBuffer: 2 * 1024 * 1024,
|
||||
windowsHide: true,
|
||||
env: calibreEnv(env,configDir,tmpDir),
|
||||
}, execFileImpl);
|
||||
} catch (e) {
|
||||
if (e?.code === 'ENOENT') {
|
||||
info('cover.calibre.unavailable', 'Calibre cover resolver is not installed; continuing with native providers.', {bookId:book.id,title:book.title,bin});
|
||||
return { status:'unavailable', reason:'not-installed' };
|
||||
}
|
||||
lastDetail=trimLog(e?.stderr || e?.stdout || e?.message);
|
||||
if (!fs.existsSync(coverPath)) {
|
||||
info('cover.calibre.attempt.none', 'Calibre identify/cover query did not return a cover.', {
|
||||
bookId:book.id,title:book.title,strategy:plan.name,exitCode:e?.code ?? null, detail:lastDetail || undefined,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
result = { stdout:e?.stdout || '', stderr:e?.stderr || '' };
|
||||
}
|
||||
|
||||
if (!fs.existsSync(coverPath)) {
|
||||
lastDetail=trimLog(result?.stderr || result?.stdout);
|
||||
info('cover.calibre.attempt.none', 'Calibre identify/cover query completed without a cover.', {
|
||||
bookId:book.id,title:book.title,strategy:plan.name,detail:lastDetail || undefined,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const buffer = fs.readFileSync(coverPath);
|
||||
info('cover.calibre.result', 'Calibre returned a cover candidate.', {
|
||||
bookId:book.id,title:book.title,strategy:plan.name,bytes:buffer.length,durationMs:Date.now()-started,
|
||||
});
|
||||
return { status:'matched', buffer, strategy:plan.name, detail:trimLog(result?.stderr || result?.stdout) };
|
||||
}
|
||||
|
||||
const direct=await runDirectCoverStage(book,{tmpDir,configDir,execFileImpl,debugBin,timeoutMs,env});
|
||||
if(direct.status==='matched') return direct;
|
||||
lastDetail=direct.detail||lastDetail;
|
||||
|
||||
// Even when Calibre cannot download a cover, its identify providers may
|
||||
// know a stable Google Books id or ISBN. Preserve those identifiers so the
|
||||
// kovi resolver can use deterministic provider URLs instead of relying
|
||||
// on Google Images scraping.
|
||||
const metadataCandidates=await runMetadataIdentifyStage(book,{tmpDir,configDir,execFileImpl,debugBin,timeoutMs,env});
|
||||
|
||||
info('cover.calibre.none', 'Calibre did not return a cover after identify and cover-only query plans.', {
|
||||
bookId:book.id,title:book.title,durationMs:Date.now()-started,metadataCandidates:metadataCandidates.length,detail:lastDetail || undefined,
|
||||
});
|
||||
return { status:'none', reason:'no-cover', detail:lastDetail, metadataCandidates };
|
||||
} catch (e) {
|
||||
warn('cover.calibre.error', 'Calibre cover lookup failed; continuing with native providers.', {
|
||||
bookId:book.id,title:book.title,error:String(e?.message || e),durationMs:Date.now()-started,
|
||||
});
|
||||
return { status:'none', reason:'error', detail:String(e?.message || e) };
|
||||
} finally {
|
||||
fs.rmSync(tmpDir, { recursive:true, force:true });
|
||||
}
|
||||
}
|
||||
+726
@@ -0,0 +1,726 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { cleanText, nowIso, randomId } from './security.mjs';
|
||||
import { info, warn } from './logger.mjs';
|
||||
import { fetchCoverWithCalibre } from './calibre.mjs';
|
||||
|
||||
const VERSION = '2026.09.25';
|
||||
const MATCH_THRESHOLD = 0.72;
|
||||
const NETWORK_RETRIES = 2;
|
||||
const COVER_WORKER_CONCURRENCY = Math.max(1, Math.min(4, Number.parseInt(process.env.COVER_WORKER_CONCURRENCY || '2', 10) || 2));
|
||||
const GOOGLE_DUMMY_MD5 = new Set(['0de4383ebad0adad5eeb8975cd796657','a64fa89d7ebc97075c1d363fc5fea71f']);
|
||||
const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
|
||||
const normalize = (s='') => String(s).toLocaleLowerCase().normalize('NFKD').replace(/[\u0300-\u036f]/g,'').replace(/[^\p{L}\p{N}]+/gu,' ').trim();
|
||||
function tokens(s){ return new Set(normalize(s).split(' ').filter(Boolean)); }
|
||||
function jaccard(a,b){ const A=tokens(a),B=tokens(b); if(!A.size||!B.size)return 0; let i=0; for(const x of A) if(B.has(x)) i++; return i/(A.size+B.size-i); }
|
||||
function containment(a,b){ const A=tokens(a),B=tokens(b); if(!A.size||!B.size)return 0; let i=0; for(const x of A) if(B.has(x)) i++; return i/Math.min(A.size,B.size); }
|
||||
|
||||
const LANG = {
|
||||
it:'it', ita:'it', italian:'it', italiano:'it',
|
||||
en:'en', eng:'en', english:'en',
|
||||
fr:'fr', fre:'fr', fra:'fr', french:'fr', francais:'fr', français:'fr',
|
||||
de:'de', ger:'de', deu:'de', german:'de', deutsch:'de',
|
||||
es:'es', spa:'es', spanish:'es', espanol:'es', español:'es',
|
||||
pt:'pt', por:'pt', portuguese:'pt', portugues:'pt', português:'pt',
|
||||
ja:'ja', jpn:'ja', japanese:'ja',
|
||||
};
|
||||
const OL_LANG = {it:'ita',en:'eng',fr:'fre',de:'ger',es:'spa',pt:'por',ja:'jpn'};
|
||||
function lang2(value){
|
||||
const raw=String(value||'').trim().toLocaleLowerCase();
|
||||
if(!raw) return null;
|
||||
const baseRaw=raw.split(/[-_]/)[0];
|
||||
const n=normalize(raw).replace(/\s+/g,'');
|
||||
const base=normalize(baseRaw).replace(/\s+/g,'');
|
||||
return LANG[raw] || LANG[n] || LANG[baseRaw] || LANG[base] || (base.length===2 ? base : null);
|
||||
}
|
||||
function langMatches(bookLang,candidateLangs=[]){
|
||||
const two=lang2(bookLang); if(!two) return false;
|
||||
const wanted=new Set([two,OL_LANG[two]].filter(Boolean));
|
||||
return candidateLangs.some(x=>wanted.has(normalize(x)) || lang2(x)===two);
|
||||
}
|
||||
|
||||
function authorSimilarity(bookAuthor, candidateAuthors) {
|
||||
const ba=normalize(bookAuthor||'');
|
||||
if(!ba || !candidateAuthors?.length) return 0;
|
||||
const authors=candidateAuthors.map(normalize).filter(Boolean);
|
||||
if(authors.some(a=>a===ba || a.includes(ba) || ba.includes(a))) return 1;
|
||||
const bookTokens=[...tokens(ba)];
|
||||
for(const a of authors){
|
||||
const at=[...tokens(a)];
|
||||
if(at.length>=2 && at.every(t=>bookTokens.includes(t))) return 0.96;
|
||||
}
|
||||
return Math.max(0,...authors.map(a=>Math.max(jaccard(ba,a), containment(ba,a)*0.9)));
|
||||
}
|
||||
|
||||
export function scoreCandidate(book, doc) {
|
||||
if (!doc?.cover_i && !doc?.cover_olid && !doc?.cover_isbn && !doc?.cover_url) return 0;
|
||||
const bt=normalize(book.title), dt=normalize(doc.title || '');
|
||||
const jac=jaccard(bt,dt), contain=containment(bt,dt);
|
||||
let score = bt && bt===dt ? 0.68 : Math.max(0.52*jac, 0.48*contain);
|
||||
// Edition catalogues often reorder a series name and translated volume title,
|
||||
// e.g. "Nevernight. I grandi giochi" vs "I grandi giochi. Nevernight".
|
||||
// If the complete token sets are identical, the title is effectively exact
|
||||
// even though word order/punctuation differ. This is safe enough to clear the
|
||||
// normal threshold without requiring author metadata on the edition record.
|
||||
const btokens=tokens(bt), dtokens=tokens(dt);
|
||||
if(jac===1 && btokens.size===dtokens.size && btokens.size>=3) score=Math.max(score,0.76);
|
||||
else if (contain >= 0.95 && jac >= 0.72) score = Math.max(score, 0.58);
|
||||
const authorScore=authorSimilarity(book.authors,doc.author_name||doc.authors||[]);
|
||||
if(authorScore>=0.92) score += 0.28;
|
||||
else score += 0.18*authorScore;
|
||||
if(book.language && Array.isArray(doc.language) && langMatches(book.language,doc.language)) score+=0.06;
|
||||
else if(book.language && doc.language && lang2(book.language)===lang2(doc.language)) score+=0.06;
|
||||
return Math.min(1,score);
|
||||
}
|
||||
|
||||
function candidateDocs(data) {
|
||||
const out=[];
|
||||
for(const work of data?.docs||[]){
|
||||
if(work.cover_i) out.push(work);
|
||||
const editions=work?.editions?.docs;
|
||||
if(Array.isArray(editions)) for(const edition of editions){
|
||||
const cover_i=edition.cover_i || work.cover_i;
|
||||
const cover_olid=String(edition.key||'').replace(/^\/books\//,'') || null;
|
||||
if(!cover_i && !cover_olid) continue;
|
||||
out.push({
|
||||
...work,
|
||||
title:edition.title || work.title,
|
||||
cover_i,
|
||||
language:edition.language || work.language,
|
||||
edition_key:edition.key,
|
||||
cover_olid,
|
||||
});
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function simplifiedTitle(title='') {
|
||||
return String(title)
|
||||
.replace(/\s*[\[(].*?[\])]\s*$/,'')
|
||||
.replace(/\s*[:–—-]\s+[^:–—-]{3,}$/,'')
|
||||
.trim();
|
||||
}
|
||||
|
||||
function titleVariants(title='') {
|
||||
const raw=String(title||'').trim();
|
||||
const out=[];
|
||||
const push=v=>{v=String(v||'').trim(); if(v && !out.some(x=>normalize(x)===normalize(v))) out.push(v);};
|
||||
push(raw);
|
||||
push(simplifiedTitle(raw));
|
||||
// Italian publishers often prefix the translated volume title with the series,
|
||||
// e.g. "Nevernight. I grandi giochi" while catalogues store "I grandi giochi".
|
||||
const dot=raw.match(/^.{2,32}\.\s+(.{3,})$/);
|
||||
if(dot) { push(dot[1]); push(simplifiedTitle(dot[1])); }
|
||||
const colon=raw.match(/^.{2,32}:\s+(.{3,})$/);
|
||||
if(colon) push(colon[1]);
|
||||
return out.slice(0,4);
|
||||
}
|
||||
|
||||
function quoteSolr(value=''){ return `"${String(value).replace(/["\\]/g,' ').replace(/\s+/g,' ').trim()}"`; }
|
||||
function openLibraryStrategies(book){
|
||||
const author=String(book.authors||'').trim();
|
||||
const out=[];
|
||||
const push=(name,q)=>{ if(q && !out.some(x=>x.q===q)) out.push({name,q}); };
|
||||
for(const [i,title] of titleVariants(book.title).entries()) {
|
||||
const label=i===0?'edition-title':`title-variant-${i}`;
|
||||
if(author) push(`${label}-author`, `${quoteSolr(title)} author:${quoteSolr(author)}`);
|
||||
push(label, quoteSolr(title));
|
||||
}
|
||||
const raw=String(book.title||'').trim();
|
||||
if(raw&&author) push('relaxed-title-author', `${raw} author:${quoteSolr(author)}`);
|
||||
return out.slice(0,8);
|
||||
}
|
||||
|
||||
async function fetchWithRetry(url, options, fetchImpl, { attempts=NETWORK_RETRIES, label='request', timeoutMs=7000 }={}) {
|
||||
let lastError;
|
||||
for(let attempt=0; attempt<attempts; attempt++){
|
||||
try {
|
||||
const resp=await fetchImpl(url,{...options,signal:AbortSignal.timeout(timeoutMs)});
|
||||
if(resp.ok || ![408,425,429,500,502,503,504].includes(resp.status)) return resp;
|
||||
lastError=Object.assign(new Error(`${label} returned ${resp.status}`),{status:resp.status});
|
||||
} catch(e) {
|
||||
lastError=e;
|
||||
if(e?.name==='AbortError' || e?.name==='TimeoutError') lastError=e;
|
||||
}
|
||||
if(attempt<attempts-1) await sleep(180 * (attempt+1));
|
||||
}
|
||||
throw lastError || new Error(`${label} failed`);
|
||||
}
|
||||
|
||||
async function searchOpenLibrary(book, strategy, fetchImpl) {
|
||||
const params=new URLSearchParams({
|
||||
q:strategy.q,
|
||||
fields:'key,title,author_name,cover_i,language,first_publish_year,editions,editions.key,editions.title,editions.cover_i,editions.language',
|
||||
limit:'20'
|
||||
});
|
||||
const preferred=lang2(book.language);
|
||||
if(preferred) params.set('lang',preferred);
|
||||
const searchUrl=`https://openlibrary.org/search.json?${params}`;
|
||||
const resp=await fetchWithRetry(searchUrl,{headers:{'User-Agent':`kovi/${VERSION} (+self-hosted-reader-dashboard)`}},fetchImpl,{label:'Open Library search',timeoutMs:6500});
|
||||
if(!resp.ok) throw new Error(`Open Library search returned ${resp.status}`);
|
||||
return { data:await resp.json(), searchUrl };
|
||||
}
|
||||
|
||||
function googleStrategies(book){
|
||||
const author=String(book.authors||'').trim();
|
||||
const out=[];
|
||||
const push=(name,q)=>{if(q&&!out.some(x=>x.q===q))out.push({name,q});};
|
||||
if(book.isbn) push('google-isbn',`isbn:${book.isbn}`);
|
||||
for(const [i,title] of titleVariants(book.title).entries()) {
|
||||
const label=i===0?'google-title':`google-title-variant-${i}`;
|
||||
if(author) push(`${label}-author`, `intitle:${quoteSolr(title)} inauthor:${quoteSolr(author)}`);
|
||||
push(label, `intitle:${quoteSolr(title)}`);
|
||||
}
|
||||
return out.slice(0,8);
|
||||
}
|
||||
|
||||
function googleCandidates(data){
|
||||
const out=[];
|
||||
for(const item of data?.items||[]){
|
||||
const v=item?.volumeInfo||{};
|
||||
const identifiers=Array.isArray(v.industryIdentifiers)?v.industryIdentifiers:[];
|
||||
const isbn13=identifiers.find(x=>x?.type==='ISBN_13')?.identifier;
|
||||
const isbn10=identifiers.find(x=>x?.type==='ISBN_10')?.identifier;
|
||||
const isbn=String(isbn13||isbn10||'').replace(/[^0-9Xx]/g,'');
|
||||
const imageLinks=v.imageLinks||{};
|
||||
const coverUrl=imageLinks.extraLarge||imageLinks.large||imageLinks.medium||imageLinks.small||imageLinks.thumbnail||imageLinks.smallThumbnail||null;
|
||||
if(!isbn && !coverUrl) continue;
|
||||
out.push({
|
||||
title:v.title||'',
|
||||
author_name:Array.isArray(v.authors)?v.authors:[],
|
||||
language:v.language||null,
|
||||
cover_isbn:isbn||null,
|
||||
cover_url:coverUrl,
|
||||
google_id:item.id,
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
async function searchGoogleBooks(book,strategy,fetchImpl,apiKey){
|
||||
const params=new URLSearchParams({q:strategy.q,maxResults:'20',printType:'books',projection:'lite'});
|
||||
if(apiKey) params.set('key',apiKey);
|
||||
const preferred=lang2(book.language); if(preferred) params.set('langRestrict',preferred);
|
||||
const url=`https://www.googleapis.com/books/v1/volumes?${params}`;
|
||||
const resp=await fetchWithRetry(url,{headers:{'User-Agent':`kovi/${VERSION}`}},fetchImpl,{label:'Google Books search',timeoutMs:6500});
|
||||
if(!resp.ok) throw new Error(`Google Books search returned ${resp.status}`);
|
||||
return {data:await resp.json(),searchUrl:url};
|
||||
}
|
||||
|
||||
export function detectImageType(buf){
|
||||
if(buf.length>3 && buf[0]===0xff&&buf[1]===0xd8&&buf[2]===0xff) return {ext:'jpg',mime:'image/jpeg'};
|
||||
if(buf.length>8 && buf.subarray(0,8).equals(Buffer.from([137,80,78,71,13,10,26,10]))) return {ext:'png',mime:'image/png'};
|
||||
if(buf.length>12 && buf.toString('ascii',0,4)==='RIFF' && buf.toString('ascii',8,12)==='WEBP') return {ext:'webp',mime:'image/webp'};
|
||||
if(buf.length>6 && (buf.toString('ascii',0,6)==='GIF87a' || buf.toString('ascii',0,6)==='GIF89a')) return {ext:'gif',mime:'image/gif'};
|
||||
return null;
|
||||
}
|
||||
|
||||
function googleHostAllowed(url) {
|
||||
try {
|
||||
const host=new URL(url).hostname.toLowerCase();
|
||||
return host==='books.google.com' || host==='books.googleapis.com' || host.endsWith('.googleusercontent.com');
|
||||
} catch { return false; }
|
||||
}
|
||||
|
||||
async function fetchValidatedImage(url, provider, fetchImpl) {
|
||||
const target=provider==='googlebooks' ? String(url).replace(/^http:/,'https:') : String(url);
|
||||
if(provider==='googlebooks' && !googleHostAllowed(target)) throw new Error('Google Books returned an unexpected cover host');
|
||||
const img=await fetchWithRetry(target,{headers:{'User-Agent':`kovi/${VERSION}`},redirect:'follow'},fetchImpl,{label:`${provider} cover fetch`,timeoutMs:8000});
|
||||
if(!img.ok) {
|
||||
const err=new Error(`${provider} cover fetch returned ${img.status}`);
|
||||
err.status=img.status;
|
||||
throw err;
|
||||
}
|
||||
if(provider==='googlebooks' && img.url && !googleHostAllowed(img.url)) throw new Error('Google Books cover redirected to an unexpected host');
|
||||
const len=Number(img.headers.get('content-length')||0);
|
||||
if(len>5_000_000) throw new Error('Cover image too large');
|
||||
const buf=Buffer.from(await img.arrayBuffer());
|
||||
if(buf.length>5_000_000) throw new Error('Cover image too large');
|
||||
if(provider==='googlebooks'){
|
||||
const md5=createHash('md5').update(buf).digest('hex');
|
||||
if(GOOGLE_DUMMY_MD5.has(md5)) throw new Error('Google Books returned a known dummy cover image');
|
||||
}
|
||||
const type=detectImageType(buf);
|
||||
if(!type){
|
||||
const err=new Error('Cover provider returned unsupported image data');
|
||||
err.coverMeta={contentType:cleanText(img.headers.get('content-type'),100),bytes:buf.length};
|
||||
throw err;
|
||||
}
|
||||
return {buf,type,url:target};
|
||||
}
|
||||
|
||||
function cacheBuffer(book, dataDir, buf, type, suffix='') {
|
||||
const coversDir=path.join(dataDir,'covers'); fs.mkdirSync(coversDir,{recursive:true});
|
||||
// Cover paths are immutable so cover history can safely point at an older image.
|
||||
// The random suffix also eliminates browser-cache ambiguity after replacements.
|
||||
const filename=`${book.id}${suffix}-${Date.now().toString(36)}-${randomId(4)}.${type.ext}`;
|
||||
const finalPath=path.join(coversDir,filename);
|
||||
const tempPath=path.join(coversDir,`.${filename}.${randomId(6)}.tmp`);
|
||||
fs.writeFileSync(tempPath,buf,{mode:0o644,flag:'wx'});
|
||||
fs.renameSync(tempPath,finalPath);
|
||||
return `/covers/${filename}`;
|
||||
}
|
||||
|
||||
function openLibraryCandidateUrls(candidate){
|
||||
const urls=[];
|
||||
const push=u=>{if(u&&!urls.includes(u))urls.push(u);};
|
||||
if(candidate.cover_i){
|
||||
push(`https://covers.openlibrary.org/b/id/${Number(candidate.cover_i)}-L.jpg?default=false`);
|
||||
push(`https://covers.openlibrary.org/b/id/${Number(candidate.cover_i)}-M.jpg?default=false`);
|
||||
}
|
||||
if(candidate.cover_olid) push(`https://covers.openlibrary.org/b/olid/${encodeURIComponent(candidate.cover_olid)}-L.jpg?default=false`);
|
||||
if(candidate.cover_isbn) push(`https://covers.openlibrary.org/b/isbn/${encodeURIComponent(candidate.cover_isbn)}-L.jpg?default=false`);
|
||||
return urls;
|
||||
}
|
||||
|
||||
async function tryCandidateDownload(book,candidate,provider,dataDir,fetchImpl,{strategy='unknown'}={}){
|
||||
const attempts=[];
|
||||
const urls=[];
|
||||
if(provider==='openlibrary') for(const url of openLibraryCandidateUrls(candidate)) urls.push({url,fetchProvider:'openlibrary',source:'openlibrary'});
|
||||
if(provider==='googlebooks') {
|
||||
// Prefer ISBN through Open Library when Google found the bibliographic record,
|
||||
// but use Google's own returned cover as a validated/cacheable fallback.
|
||||
for(const url of openLibraryCandidateUrls(candidate)) urls.push({url,fetchProvider:'openlibrary',source:'openlibrary-via-googlebooks'});
|
||||
if(candidate.cover_url) urls.push({url:candidate.cover_url,fetchProvider:'googlebooks',source:'googlebooks'});
|
||||
for(const url of candidate.cover_urls||[]) if(url) urls.push({url,fetchProvider:'googlebooks',source:'googlebooks-via-calibre-id'});
|
||||
}
|
||||
for(const item of urls){
|
||||
try{
|
||||
const {buf,type}=await fetchValidatedImage(item.url,item.fetchProvider,fetchImpl);
|
||||
return {ok:true,path:cacheBuffer(book,dataDir,buf,type),source:item.source};
|
||||
}catch(e){
|
||||
const failure={provider:item.fetchProvider,strategy,error:e.message,status:e.status||null,...(e.coverMeta||{})};
|
||||
attempts.push(failure);
|
||||
warn('cover.download.rejected','Rejected a cover image and will continue with other candidates.',{bookId:book.id,title:book.title,...failure});
|
||||
}
|
||||
}
|
||||
return {ok:false,attempts};
|
||||
}
|
||||
|
||||
function calibreMetadataDocs(calibre){
|
||||
const out=[];
|
||||
for(const item of calibre?.metadataCandidates||[]){
|
||||
const ids=item?.identifiers||{};
|
||||
const google=String(ids.google||ids.Google||'').trim();
|
||||
const isbn=String(ids.isbn||ids.ISBN||'').replace(/[^0-9Xx]/g,'');
|
||||
const cover_urls=[];
|
||||
if(google){
|
||||
const id=encodeURIComponent(google);
|
||||
// This is the same stable Google Books cover endpoint used by Calibre's
|
||||
// Google metadata source. Try both zoom levels because some records only
|
||||
// expose one of them.
|
||||
cover_urls.push(`https://books.google.com/books?id=${id}&printsec=frontcover&img=1&zoom=0`);
|
||||
cover_urls.push(`https://books.google.com/books?id=${id}&printsec=frontcover&img=1&zoom=1`);
|
||||
}
|
||||
if(!google && !isbn) continue;
|
||||
out.push({
|
||||
title:item.title||'',
|
||||
author_name:Array.isArray(item.authors)?item.authors:[],
|
||||
language:Array.isArray(item.languages)?item.languages:item.languages||null,
|
||||
cover_isbn:isbn||null,
|
||||
google_id:google||null,
|
||||
cover_urls,
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
async function tryDirectIsbn(book,dataDir,fetchImpl){
|
||||
if(!book.isbn) return null;
|
||||
const candidate={title:book.title,author_name:[book.authors].filter(Boolean),language:book.language,cover_isbn:book.isbn};
|
||||
const dl=await tryCandidateDownload(book,candidate,'openlibrary',dataDir,fetchImpl,{strategy:'isbn-direct'});
|
||||
const attempt={provider:'openlibrary',strategy:'isbn-direct',isbn:book.isbn,candidates:1,bestScore:1,downloaded:dl.ok};
|
||||
info('cover.lookup.attempt','Cover lookup strategy completed.',{bookId:book.id,title:book.title,...attempt});
|
||||
return dl.ok ? {status:'matched',path:dl.path,source:dl.source,score:1,matchedTitle:book.title,strategy:'isbn-direct',trace:[attempt]} : {status:'none',score:1,trace:[attempt,...dl.attempts]};
|
||||
}
|
||||
|
||||
async function tryGoogleIsbn(book,dataDir,fetchImpl,apiKey){
|
||||
if(!book.isbn) return null;
|
||||
try {
|
||||
const strategy={name:'google-isbn',q:`isbn:${book.isbn}`};
|
||||
const {data}=await searchGoogleBooks(book,strategy,fetchImpl,apiKey);
|
||||
const docs=googleCandidates(data);
|
||||
const ranked=docs.map(d=>({doc:d,score:scoreCandidate(book,d)})).sort((a,b)=>b.score-a.score);
|
||||
const attempt={provider:'googlebooks',strategy:'google-isbn',isbn:book.isbn,results:Number(data?.totalItems||0),candidates:docs.length,bestScore:ranked[0]?.score||0};
|
||||
for(const candidate of ranked.slice(0,3)){
|
||||
// ISBN queries are deterministic enough to accept a returned cover even if
|
||||
// translated title metadata differs, while still preferring the best rank.
|
||||
const dl=await tryCandidateDownload(book,candidate.doc,'googlebooks',dataDir,fetchImpl,{strategy:'google-isbn'});
|
||||
if(dl.ok) return {status:'matched',path:dl.path,source:dl.source,score:Math.max(candidate.score,0.95),matchedTitle:candidate.doc.title||book.title,strategy:'google-isbn',trace:[attempt]};
|
||||
attempt.downloadFailures=(attempt.downloadFailures||0)+dl.attempts.length;
|
||||
}
|
||||
return {status:'none',score:attempt.bestScore,trace:[attempt]};
|
||||
} catch(e) {
|
||||
return {status:'none',score:0,trace:[{provider:'googlebooks',strategy:'google-isbn',error:e.message,status:e.status||null}]};
|
||||
}
|
||||
}
|
||||
|
||||
async function searchNativePair(book, olStrategy, googleStrategy, fetchImpl, apiKey){
|
||||
const [ol,google]=await Promise.allSettled([
|
||||
olStrategy ? searchOpenLibrary(book,olStrategy,fetchImpl) : Promise.resolve(null),
|
||||
googleStrategy ? searchGoogleBooks(book,googleStrategy,fetchImpl,apiKey) : Promise.resolve(null),
|
||||
]);
|
||||
return {ol,google};
|
||||
}
|
||||
|
||||
export async function resolveCover(book, { dataDir, fetchImpl = fetch, googleBooksApiKey = process.env.GOOGLE_BOOKS_API_KEY || '', calibreResolver = fetchCoverWithCalibre } = {}) {
|
||||
const trace=[];
|
||||
let bestScore=0;
|
||||
|
||||
// Stable identifiers are the quickest and most reliable path. Open Library's
|
||||
// ISBN cover endpoint is a single request; if it misses, Google Books gets a
|
||||
// deterministic ISBN query before we fall back to fuzzy title matching.
|
||||
if(book.isbn){
|
||||
const direct=await tryDirectIsbn(book,dataDir,fetchImpl);
|
||||
if(direct?.status==='matched') return direct;
|
||||
if(direct) { trace.push(...direct.trace); bestScore=Math.max(bestScore,direct.score||0); }
|
||||
const googleIsbn=await tryGoogleIsbn(book,dataDir,fetchImpl,googleBooksApiKey);
|
||||
if(googleIsbn?.status==='matched') { googleIsbn.trace=[...trace,...(googleIsbn.trace||[])]; return googleIsbn; }
|
||||
if(googleIsbn) { trace.push(...(googleIsbn.trace||[])); bestScore=Math.max(bestScore,googleIsbn.score||0); }
|
||||
}
|
||||
|
||||
const olStrategies=openLibraryStrategies(book);
|
||||
const googlePlans=googleStrategies({...book,isbn:null}).filter(x=>x.name!=='google-isbn');
|
||||
const maxNative=Math.max(olStrategies.length,googlePlans.length);
|
||||
|
||||
// Query the two lightweight catalogues together. This removes the old
|
||||
// Open-Library-first waterfall (and its per-query sleeps), so a Google Books
|
||||
// hit can arrive without waiting through every Open Library title variant.
|
||||
for(let i=0;i<maxNative;i++){
|
||||
const olStrategy=olStrategies[i]||null;
|
||||
const googleStrategy=googlePlans[i]||null;
|
||||
const pair=await searchNativePair(book,olStrategy,googleStrategy,fetchImpl,googleBooksApiKey);
|
||||
const groups=[];
|
||||
|
||||
if(olStrategy){
|
||||
if(pair.ol.status==='fulfilled'){
|
||||
const data=pair.ol.value?.data;
|
||||
const docs=candidateDocs(data);
|
||||
const ranked=docs.map(d=>({doc:d,score:scoreCandidate(book,d)})).sort((a,b)=>b.score-a.score);
|
||||
const top=ranked[0]; bestScore=Math.max(bestScore,top?.score||0);
|
||||
const attempt={provider:'openlibrary',strategy:olStrategy.name,results:Number(data?.numFound||data?.docs?.length||0),candidates:docs.length,bestScore:top?.score||0,matchedTitle:top?.doc?.title||null};
|
||||
trace.push(attempt); groups.push({provider:'openlibrary',strategy:olStrategy.name,ranked});
|
||||
info('cover.lookup.attempt','Cover search strategy completed.',{bookId:book.id,title:book.title,...attempt,bestScore:Number((attempt.bestScore||0).toFixed(3))});
|
||||
} else trace.push({provider:'openlibrary',strategy:olStrategy.name,error:pair.ol.reason?.message||'request failed',status:pair.ol.reason?.status||null});
|
||||
}
|
||||
if(googleStrategy){
|
||||
if(pair.google.status==='fulfilled'){
|
||||
const data=pair.google.value?.data;
|
||||
const docs=googleCandidates(data);
|
||||
const ranked=docs.map(d=>({doc:d,score:scoreCandidate(book,d)})).sort((a,b)=>b.score-a.score);
|
||||
const top=ranked[0]; bestScore=Math.max(bestScore,top?.score||0);
|
||||
const attempt={provider:'googlebooks',strategy:googleStrategy.name,results:Number(data?.totalItems||0),candidates:docs.length,bestScore:top?.score||0,matchedTitle:top?.doc?.title||null};
|
||||
trace.push(attempt); groups.push({provider:'googlebooks',strategy:googleStrategy.name,ranked});
|
||||
info('cover.lookup.attempt','Cover search strategy completed.',{bookId:book.id,title:book.title,...attempt,bestScore:Number((attempt.bestScore||0).toFixed(3))});
|
||||
} else trace.push({provider:'googlebooks',strategy:googleStrategy.name,error:pair.google.reason?.message||'request failed',status:pair.google.reason?.status||null});
|
||||
}
|
||||
|
||||
// Try the strongest candidates across both providers first. This avoids
|
||||
// wasting downloads on a merely acceptable result when another catalogue
|
||||
// already returned an exact title/author match in the same round.
|
||||
const candidates=groups.flatMap(g=>g.ranked.slice(0,4).map(x=>({...x,provider:g.provider,strategy:g.strategy})))
|
||||
.filter(x=>x.score>=MATCH_THRESHOLD).sort((a,b)=>b.score-a.score);
|
||||
for(const candidate of candidates.slice(0,6)){
|
||||
const dl=await tryCandidateDownload(book,candidate.doc,candidate.provider,dataDir,fetchImpl,{strategy:candidate.strategy});
|
||||
if(dl.ok) return {status:'matched',path:dl.path,source:dl.source,score:candidate.score,matchedTitle:candidate.doc.title,strategy:candidate.strategy,trace};
|
||||
trace.push(...dl.attempts);
|
||||
}
|
||||
}
|
||||
|
||||
// Heavyweight federation is deliberately last. It is excellent for difficult
|
||||
// books, but spawning Calibre is much slower than the native HTTP catalogues.
|
||||
if (calibreResolver && (fetchImpl === fetch || calibreResolver !== fetchCoverWithCalibre)) {
|
||||
const calibre = await calibreResolver(book,{dataDir});
|
||||
trace.push({provider:'calibre',strategy:'metadata-source-federation',status:calibre?.status||'unknown'});
|
||||
if(calibre?.status==='matched' && calibre.buffer){
|
||||
const type=detectImageType(calibre.buffer);
|
||||
if(type && calibre.buffer.length<=5_000_000){
|
||||
const coverPath=cacheBuffer(book,dataDir,calibre.buffer,type,'-calibre');
|
||||
info('cover.calibre.matched','Accepted Calibre-selected cover.',{bookId:book.id,title:book.title,bytes:calibre.buffer.length,mime:type.mime,calibreStrategy:calibre.strategy||undefined});
|
||||
return {status:'matched',path:coverPath,source:'calibre',score:1,matchedTitle:book.title,strategy:`calibre:${calibre.strategy||'metadata-sources'}`,trace};
|
||||
}
|
||||
warn('cover.calibre.rejected','Calibre returned an unsupported or oversized image; continuing with identifier recovery.',{bookId:book.id,title:book.title,bytes:calibre.buffer.length});
|
||||
}
|
||||
|
||||
const metaDocs=calibreMetadataDocs(calibre);
|
||||
const ranked=metaDocs.map(d=>({doc:d,score:scoreCandidate(book,{...d,cover_url:d.cover_urls?.[0]||null})})).sort((a,b)=>b.score-a.score);
|
||||
if(ranked.length){
|
||||
const top=ranked[0];
|
||||
const attempt={provider:'calibre-identifiers',strategy:'google-id-or-isbn',candidates:ranked.length,bestScore:top.score,matchedTitle:top.doc.title||null,googleId:Boolean(top.doc.google_id),isbn:Boolean(top.doc.cover_isbn)};
|
||||
trace.push(attempt);
|
||||
for(const rankedCandidate of ranked.slice(0,5)){
|
||||
if(rankedCandidate.score<MATCH_THRESHOLD) break;
|
||||
const dl=await tryCandidateDownload(book,rankedCandidate.doc,'googlebooks',dataDir,fetchImpl,{strategy:'calibre-identifiers'});
|
||||
if(dl.ok) return {status:'matched',path:dl.path,source:dl.source,score:rankedCandidate.score,matchedTitle:rankedCandidate.doc.title,strategy:'calibre-identifiers',trace};
|
||||
trace.push(...dl.attempts);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { status:'none', score:bestScore, trace };
|
||||
}
|
||||
|
||||
function coverStatusForSource(source, fallback='matched') {
|
||||
return source==='manual-upload' ? 'manual' : fallback;
|
||||
}
|
||||
|
||||
function recordCoverSelection(db, book, {path:coverPath,source,status='matched',strategy=null,score=null,matchedTitle=null}={}) {
|
||||
if(!coverPath) throw new Error('A selected cover must have a local path.');
|
||||
const ts=nowIso();
|
||||
// openAppDb backfills old current covers, but keep this defensive for tests and
|
||||
// databases opened before the migration completed.
|
||||
if(book.cover_path && !db.prepare(`SELECT 1 ok FROM book_covers WHERE book_id=? AND path=?`).get(book.id,book.cover_path)){
|
||||
db.prepare(`INSERT INTO book_covers(id,book_id,path,source,strategy,score,matched_title,cover_status,selected,created_at) VALUES(?,?,?,?,NULL,NULL,NULL,?,1,?)`)
|
||||
.run(randomId(16),book.id,book.cover_path,book.cover_source,book.cover_status||coverStatusForSource(book.cover_source),book.cover_checked_at||book.updated_at||ts);
|
||||
}
|
||||
db.prepare(`UPDATE book_covers SET selected=0 WHERE book_id=?`).run(book.id);
|
||||
let existing=db.prepare(`SELECT id FROM book_covers WHERE book_id=? AND path=?`).get(book.id,coverPath);
|
||||
let id=existing?.id;
|
||||
if(id){
|
||||
db.prepare(`UPDATE book_covers SET source=?,strategy=?,score=?,matched_title=?,cover_status=?,selected=1 WHERE id=?`)
|
||||
.run(source||null,strategy||null,score==null?null:Number(score),matchedTitle||null,status,id);
|
||||
}else{
|
||||
id=randomId(16);
|
||||
db.prepare(`INSERT INTO book_covers(id,book_id,path,source,strategy,score,matched_title,cover_status,selected,created_at) VALUES(?,?,?,?,?,?,?,?,1,?)`)
|
||||
.run(id,book.id,coverPath,source||null,strategy||null,score==null?null:Number(score),matchedTitle||null,status,ts);
|
||||
}
|
||||
db.prepare(`UPDATE books SET cover_status=?,cover_path=?,cover_source=?,cover_retry_mode=NULL,cover_checked_at=?,updated_at=? WHERE id=?`)
|
||||
.run(status,coverPath,source||null,ts,ts,book.id);
|
||||
return id;
|
||||
}
|
||||
|
||||
export function getBookCoverState(db, bookId) {
|
||||
const book=db.prepare(`SELECT id,title,cover_status,cover_path,cover_source,cover_checked_at FROM books WHERE id=?`).get(bookId);
|
||||
if(!book) return null;
|
||||
const history=db.prepare(`SELECT id,path,source,strategy,score,matched_title,cover_status,selected,created_at FROM book_covers WHERE book_id=? ORDER BY selected DESC,created_at DESC LIMIT 30`).all(bookId);
|
||||
const candidates=db.prepare(`SELECT id,path,source,strategy,score,title,authors,created_at FROM cover_candidates WHERE book_id=? ORDER BY score DESC,created_at DESC LIMIT 12`).all(bookId);
|
||||
const job=db.prepare(`SELECT state,reason,force_online,attempts,requested_at,started_at,finished_at,last_error,result_status,result_source FROM cover_jobs WHERE book_id=?`).get(bookId)||null;
|
||||
return {book,history,candidates,job};
|
||||
}
|
||||
|
||||
export function saveEmbeddedCover(db, dataDir, sourceMd5, buffer) {
|
||||
if(!Buffer.isBuffer(buffer)) buffer=Buffer.from(buffer||[]);
|
||||
if(buffer.length<32) throw Object.assign(new Error('Embedded cover is empty or too small.'),{status:400});
|
||||
if(buffer.length>4_000_000) throw Object.assign(new Error('Embedded cover exceeds the 4 MB limit.'),{status:413});
|
||||
const type=detectImageType(buffer);
|
||||
if(!type) throw Object.assign(new Error('Embedded cover is not a supported JPEG, PNG, WebP, or GIF image.'),{status:400});
|
||||
const book=db.prepare(`SELECT id,title,cover_status,cover_path,cover_source,cover_checked_at,updated_at FROM books WHERE source_md5=?`).get(sourceMd5);
|
||||
if(!book) throw Object.assign(new Error('Book for embedded cover was not found.'),{status:404});
|
||||
if(book.cover_status==='manual' || book.cover_source==='manual-upload') return {ignored:true,reason:'manual-cover',book};
|
||||
const coverPath=cacheBuffer(book,dataDir,buffer,type,'-koreader');
|
||||
recordCoverSelection(db,book,{path:coverPath,source:'koreader-embedded',status:'matched',strategy:'koreader-embedded'});
|
||||
return {ignored:false,bookId:book.id,title:book.title,path:coverPath,type:type.mime,bytes:buffer.length};
|
||||
}
|
||||
|
||||
export function saveManualCover(db, dataDir, bookId, buffer) {
|
||||
if(!Buffer.isBuffer(buffer)) buffer=Buffer.from(buffer||[]);
|
||||
if(buffer.length<32) throw Object.assign(new Error('Cover image is empty or too small.'),{status:400});
|
||||
if(buffer.length>4_000_000) throw Object.assign(new Error('Cover image exceeds the 4 MB limit.'),{status:413});
|
||||
const type=detectImageType(buffer);
|
||||
if(!type) throw Object.assign(new Error('Cover image must be a JPEG, PNG, WebP, or GIF image.'),{status:400});
|
||||
const book=db.prepare(`SELECT id,title,cover_status,cover_path,cover_source,cover_checked_at,updated_at FROM books WHERE id=?`).get(bookId);
|
||||
if(!book) throw Object.assign(new Error('Book not found.'),{status:404});
|
||||
const coverPath=cacheBuffer(book,dataDir,buffer,type,'-manual');
|
||||
recordCoverSelection(db,book,{path:coverPath,source:'manual-upload',status:'manual',strategy:'manual-upload'});
|
||||
// A later manual choice supersedes any queued/running online request.
|
||||
db.prepare(`DELETE FROM cover_jobs WHERE book_id=?`).run(book.id);
|
||||
return {bookId:book.id,title:book.title,path:coverPath,type:type.mime,bytes:buffer.length};
|
||||
}
|
||||
|
||||
function queueCoverJob(db,bookId,{reason='automatic',forceOnline=false}={}){
|
||||
const ts=nowIso();
|
||||
db.prepare(`INSERT INTO cover_jobs(book_id,state,reason,force_online,attempts,requested_at,started_at,finished_at,last_error,result_status,result_source)
|
||||
VALUES(?,'queued',?,?,0,?,NULL,NULL,NULL,NULL,NULL)
|
||||
ON CONFLICT(book_id) DO UPDATE SET state='queued',reason=excluded.reason,force_online=excluded.force_online,requested_at=excluded.requested_at,started_at=NULL,finished_at=NULL,last_error=NULL,result_status=NULL,result_source=NULL`)
|
||||
.run(bookId,reason,forceOnline?1:0,ts);
|
||||
}
|
||||
|
||||
export function queueOnlineCoverRetry(db, dataDir, bookId, {reason='user-retry-book',delayMs=50}={}) {
|
||||
const book=db.prepare(`SELECT id,title FROM books WHERE id=?`).get(bookId);
|
||||
if(!book) return null;
|
||||
// Do not mutate a perfectly good local/current cover while lookup is running.
|
||||
db.prepare(`UPDATE books SET cover_retry_mode=NULL WHERE id=?`).run(book.id);
|
||||
queueCoverJob(db,book.id,{reason,forceOnline:true});
|
||||
scheduleCoverWork(db,dataDir,[],{reason,delayMs});
|
||||
return book;
|
||||
}
|
||||
|
||||
export function selectCoverHistory(db,bookId,coverId){
|
||||
const book=db.prepare(`SELECT id,title,cover_status,cover_path,cover_source,cover_checked_at,updated_at FROM books WHERE id=?`).get(bookId);
|
||||
if(!book) return null;
|
||||
const cover=db.prepare(`SELECT * FROM book_covers WHERE id=? AND book_id=?`).get(coverId,bookId);
|
||||
if(!cover) return false;
|
||||
recordCoverSelection(db,book,{path:cover.path,source:cover.source,status:cover.cover_status||coverStatusForSource(cover.source),strategy:cover.strategy,score:cover.score,matchedTitle:cover.matched_title});
|
||||
db.prepare(`DELETE FROM cover_jobs WHERE book_id=?`).run(bookId);
|
||||
return true;
|
||||
}
|
||||
|
||||
export function selectCoverCandidate(db,bookId,candidateId){
|
||||
const book=db.prepare(`SELECT id,title,cover_status,cover_path,cover_source,cover_checked_at,updated_at FROM books WHERE id=?`).get(bookId);
|
||||
if(!book) return null;
|
||||
const candidate=db.prepare(`SELECT * FROM cover_candidates WHERE id=? AND book_id=?`).get(candidateId,bookId);
|
||||
if(!candidate) return false;
|
||||
recordCoverSelection(db,book,{path:candidate.path,source:candidate.source,status:'matched',strategy:candidate.strategy,score:candidate.score,matchedTitle:candidate.title});
|
||||
db.prepare(`DELETE FROM cover_jobs WHERE book_id=?`).run(bookId);
|
||||
return true;
|
||||
}
|
||||
|
||||
export async function findCoverCandidates(db,dataDir,bookId,{fetchImpl=fetch,googleBooksApiKey=process.env.GOOGLE_BOOKS_API_KEY||'',limit=6}={}){
|
||||
const book=db.prepare(`SELECT id,title,authors,language,isbn,identifiers FROM books WHERE id=?`).get(bookId);
|
||||
if(!book) return null;
|
||||
// Candidate images are local cached previews. Remove old, unselected previews
|
||||
// for this book before building the next set.
|
||||
const old=db.prepare(`SELECT path FROM cover_candidates WHERE book_id=?`).all(bookId);
|
||||
db.prepare(`DELETE FROM cover_candidates WHERE book_id=?`).run(bookId);
|
||||
for(const row of old){
|
||||
if(db.prepare(`SELECT 1 ok FROM book_covers WHERE path=?`).get(row.path)) continue;
|
||||
const local=path.join(dataDir,String(row.path||'').replace(/^\/+/,''));
|
||||
try{fs.rmSync(local,{force:true})}catch{}
|
||||
}
|
||||
|
||||
const pool=[];
|
||||
const add=(provider,strategy,docs)=>{
|
||||
for(const doc of docs||[]){
|
||||
const score=scoreCandidate(book,doc);
|
||||
if(score<0.5) continue;
|
||||
const key=`${provider}|${doc.cover_i||doc.cover_olid||doc.cover_isbn||doc.cover_url||doc.google_id||''}|${normalize(doc.title||'')}`;
|
||||
if(pool.some(x=>x.key===key)) continue;
|
||||
pool.push({key,provider,strategy,doc,score});
|
||||
}
|
||||
};
|
||||
if(book.isbn) add('openlibrary','isbn-direct',[{title:book.title,author_name:[book.authors].filter(Boolean),language:book.language,cover_isbn:book.isbn}]);
|
||||
const olPlans=openLibraryStrategies(book).slice(0,2),googlePlans=googleStrategies(book).slice(0,2);
|
||||
for(let i=0;i<Math.max(olPlans.length,googlePlans.length);i++){
|
||||
const ol=olPlans[i]||null,google=googlePlans[i]||null;
|
||||
const pair=await searchNativePair(book,ol,google,fetchImpl,googleBooksApiKey);
|
||||
if(ol && pair.ol.status==='fulfilled') add('openlibrary',ol.name,candidateDocs(pair.ol.value?.data));
|
||||
if(google && pair.google.status==='fulfilled') add('googlebooks',google.name,googleCandidates(pair.google.value?.data));
|
||||
}
|
||||
pool.sort((a,b)=>b.score-a.score);
|
||||
const rows=[];
|
||||
for(const item of pool.slice(0,Math.max(limit*3,12))){
|
||||
if(rows.length>=limit) break;
|
||||
const dl=await tryCandidateDownload(book,item.doc,item.provider,dataDir,fetchImpl,{strategy:item.strategy});
|
||||
if(!dl.ok) continue;
|
||||
const id=randomId(16),createdAt=nowIso();
|
||||
const authors=(item.doc.author_name||item.doc.authors||[]); const authorText=Array.isArray(authors)?authors.join(', '):String(authors||'');
|
||||
db.prepare(`INSERT INTO cover_candidates(id,book_id,path,source,strategy,score,title,authors,created_at) VALUES(?,?,?,?,?,?,?,?,?)`)
|
||||
.run(id,book.id,dl.path,dl.source||item.provider,item.strategy,item.score,cleanText(item.doc.title,500)||book.title,cleanText(authorText,500),createdAt);
|
||||
rows.push({id,path:dl.path,source:dl.source||item.provider,strategy:item.strategy,score:item.score,title:item.doc.title||book.title,authors:authorText,created_at:createdAt});
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
let coverWorkerRunning=false;
|
||||
let coverWorkerTimer=null;
|
||||
let coverRecoveryDone=false;
|
||||
|
||||
export async function processCoverBook(db,dataDir,book,{resolver=resolveCover,forceOnline=false}={}){
|
||||
const before=db.prepare(`SELECT cover_status,cover_path,cover_source,cover_retry_mode,cover_checked_at,updated_at FROM books WHERE id=?`).get(book.id);
|
||||
forceOnline=Boolean(forceOnline || before?.cover_retry_mode==='online');
|
||||
const localPriority=before?.cover_status==='manual' || before?.cover_source==='koreader-embedded' || before?.cover_source==='manual-upload';
|
||||
if(localPriority && !forceOnline){
|
||||
const restored=coverStatusForSource(before.cover_source);
|
||||
if(before.cover_status==='pending') db.prepare(`UPDATE books SET cover_status=?,cover_retry_mode=NULL,updated_at=? WHERE id=?`).run(restored,nowIso(),book.id);
|
||||
info('cover.lookup.local-preserved','Skipped online lookup because a higher-priority local cover is already present.',{bookId:book.id,title:book.title,coverSource:before.cover_source});
|
||||
return {status:'preserved',source:before.cover_source};
|
||||
}
|
||||
info('cover.lookup.start','Looking up cover.',{bookId:book.id,title:book.title,authors:book.authors,language:book.language,isbn:book.isbn||undefined,googleBooks:true,forceOnline});
|
||||
try{
|
||||
const r=await resolver(book,{dataDir});
|
||||
const current=db.prepare(`SELECT cover_status,cover_path,cover_source,cover_checked_at,updated_at FROM books WHERE id=?`).get(book.id);
|
||||
const changedDuringLookup=current && (current.cover_path!==before?.cover_path || current.cover_source!==before?.cover_source);
|
||||
const currentLocal=current?.cover_status==='manual' || current?.cover_source==='koreader-embedded' || current?.cover_source==='manual-upload';
|
||||
if(currentLocal && (changedDuringLookup || !forceOnline)){
|
||||
info('cover.lookup.superseded','Online cover result was superseded by a newer/higher-priority local cover.',{bookId:book.id,title:book.title,coverSource:current.cover_source});
|
||||
return {status:'superseded',source:current.cover_source};
|
||||
}
|
||||
if(forceOnline && r.status!=='matched' && current?.cover_path){
|
||||
const restored=coverStatusForSource(current.cover_source);
|
||||
db.prepare(`UPDATE books SET cover_status=?,cover_retry_mode=NULL,cover_checked_at=?,updated_at=? WHERE id=?`).run(restored,nowIso(),nowIso(),book.id);
|
||||
info('cover.lookup.none','No confident online cover match found; keeping the existing cover.',{bookId:book.id,title:book.title,coverSource:current.cover_source,bestScore:Number((r.score||0).toFixed(3)),attempts:r.trace});
|
||||
return {status:'none-kept',source:current.cover_source};
|
||||
}
|
||||
if(r.status==='matched'){
|
||||
const fullBook={...book,...current};
|
||||
recordCoverSelection(db,fullBook,{path:r.path,source:r.source,status:'matched',strategy:r.strategy,score:r.score,matchedTitle:r.matchedTitle});
|
||||
info('cover.lookup.matched','Cover matched and cached.',{bookId:book.id,title:book.title,matchedTitle:r.matchedTitle,score:Number((r.score||0).toFixed(3)),provider:r.source,strategy:r.strategy,attempts:r.trace});
|
||||
return {status:'matched',source:r.source};
|
||||
}
|
||||
db.prepare(`UPDATE books SET cover_status='none',cover_path=NULL,cover_source=NULL,cover_retry_mode=NULL,cover_checked_at=?,updated_at=? WHERE id=?`).run(nowIso(),nowIso(),book.id);
|
||||
info('cover.lookup.none','No confident cover match found.',{bookId:book.id,title:book.title,bestScore:Number((r.score||0).toFixed(3)),attempts:r.trace});
|
||||
return {status:'none',source:null};
|
||||
}catch(e){
|
||||
const current=db.prepare(`SELECT cover_status,cover_path,cover_source FROM books WHERE id=?`).get(book.id);
|
||||
if(current?.cover_path) db.prepare(`UPDATE books SET cover_status=?,cover_retry_mode=NULL,cover_checked_at=?,updated_at=? WHERE id=?`).run(coverStatusForSource(current.cover_source),nowIso(),nowIso(),book.id);
|
||||
else db.prepare(`UPDATE books SET cover_status='error',cover_retry_mode=NULL,cover_checked_at=?,updated_at=? WHERE id=?`).run(nowIso(),nowIso(),book.id);
|
||||
warn('cover.lookup.error','Cover lookup failed.',{bookId:book.id,title:book.title,error:e.message});
|
||||
return {status:'error',source:current?.cover_source||null,error:e.message};
|
||||
}
|
||||
}
|
||||
|
||||
async function runCoverBatch(db,dataDir){
|
||||
const jobs=db.prepare(`SELECT j.book_id,j.reason,j.force_online,b.id,b.title,b.authors,b.language,b.isbn,b.identifiers,b.cover_path,b.cover_source,b.cover_status
|
||||
FROM cover_jobs j JOIN books b ON b.id=j.book_id WHERE j.state='queued' ORDER BY j.requested_at LIMIT 100`).all();
|
||||
if(!jobs.length) return 0;
|
||||
let cursor=0;
|
||||
const workers=Array.from({length:Math.min(COVER_WORKER_CONCURRENCY,jobs.length)},async()=>{
|
||||
while(cursor<jobs.length){
|
||||
const job=jobs[cursor++];
|
||||
const claimed=db.prepare(`UPDATE cover_jobs SET state='running',attempts=attempts+1,started_at=?,finished_at=NULL,last_error=NULL WHERE book_id=? AND state='queued'`).run(nowIso(),job.book_id).changes;
|
||||
if(!claimed) continue;
|
||||
const outcome=await processCoverBook(db,dataDir,job,{forceOnline:Boolean(job.force_online)});
|
||||
const state=outcome.status==='error'?'error':'done';
|
||||
db.prepare(`UPDATE cover_jobs SET state=?,finished_at=?,last_error=?,result_status=?,result_source=? WHERE book_id=?`)
|
||||
.run(state,nowIso(),outcome.error||null,outcome.status||null,outcome.source||null,job.book_id);
|
||||
}
|
||||
});
|
||||
await Promise.all(workers);
|
||||
return jobs.length;
|
||||
}
|
||||
|
||||
export function scheduleCoverWork(db, dataDir, ids = [], {reason='automatic',delayMs=50}={}) {
|
||||
// Recover interrupted jobs once after a process restart. Do not rewrite a genuinely
|
||||
// running job when another import/user action schedules unrelated work.
|
||||
if(!coverRecoveryDone){db.prepare(`UPDATE cover_jobs SET state='queued',started_at=NULL WHERE state='running'`).run();coverRecoveryDone=true;}
|
||||
const legacy=db.prepare(`SELECT id,cover_path,cover_source,cover_status,cover_retry_mode FROM books WHERE cover_retry_mode='online'`).all();
|
||||
for(const row of legacy){
|
||||
const restored=row.cover_path ? coverStatusForSource(row.cover_source) : 'pending';
|
||||
db.prepare(`UPDATE books SET cover_status=?,cover_retry_mode=NULL WHERE id=?`).run(restored,row.id);
|
||||
queueCoverJob(db,row.id,{reason:'legacy-user-retry',forceOnline:true});
|
||||
}
|
||||
|
||||
const staleErrors=db.prepare(`SELECT id FROM books WHERE cover_status='error' AND (cover_checked_at IS NULL OR cover_checked_at < datetime('now','-6 hours'))`).all();
|
||||
for(const row of staleErrors){db.prepare(`UPDATE books SET cover_status='pending' WHERE id=? AND cover_path IS NULL`).run(row.id);queueCoverJob(db,row.id,{reason:'recover-error'});}
|
||||
let explicitlyQueued=0;
|
||||
for(const id of ids){
|
||||
const row=db.prepare(`SELECT id,cover_path,cover_status FROM books WHERE id=?`).get(id); if(!row) continue;
|
||||
if(!row.cover_path) db.prepare(`UPDATE books SET cover_status='pending' WHERE id=?`).run(id);
|
||||
queueCoverJob(db,id,{reason}); explicitlyQueued++;
|
||||
}
|
||||
// Any pending book without a job still deserves one; this is how imported
|
||||
// books are recovered automatically on startup.
|
||||
const pendingBooks=db.prepare(`SELECT id FROM books WHERE cover_status='pending'`).all();
|
||||
for(const row of pendingBooks) if(!db.prepare(`SELECT 1 ok FROM cover_jobs WHERE book_id=? AND state IN ('queued','running')`).get(row.id)) queueCoverJob(db,row.id,{reason:'pending-cover'});
|
||||
const pending=Number(db.prepare(`SELECT COUNT(*) n FROM cover_jobs WHERE state='queued'`).get().n);
|
||||
if(explicitlyQueued || pending) info('cover.queue','Cover work queued.',{reason,pending,requested:ids.length,delayMs,concurrency:COVER_WORKER_CONCURRENCY});
|
||||
if(coverWorkerRunning || !pending) return;
|
||||
if(coverWorkerTimer) clearTimeout(coverWorkerTimer);
|
||||
coverWorkerTimer=setTimeout(async()=>{
|
||||
coverWorkerTimer=null;
|
||||
if(coverWorkerRunning) return;
|
||||
coverWorkerRunning=true;
|
||||
info('cover.worker.start','Cover worker started.',{pending,concurrency:COVER_WORKER_CONCURRENCY});
|
||||
let processed=0;
|
||||
try{ processed=await runCoverBatch(db,dataDir); }
|
||||
finally {
|
||||
coverWorkerRunning=false;
|
||||
const remaining=Number(db.prepare(`SELECT COUNT(*) n FROM cover_jobs WHERE state='queued'`).get().n);
|
||||
info('cover.worker.stop','Cover worker finished a batch.',{processed,remaining});
|
||||
if(remaining) scheduleCoverWork(db,dataDir,[],{reason:'continue-batch',delayMs:100});
|
||||
}
|
||||
},Math.max(0,Number(delayMs)||0));
|
||||
}
|
||||
|
||||
+494
@@ -0,0 +1,494 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { DatabaseSync } from 'node:sqlite';
|
||||
import { randomInt } from 'node:crypto';
|
||||
import { nowIso, randomId, randomToken, tokenHash, safeEqualHex } from './security.mjs';
|
||||
|
||||
const dbTimeZones = new WeakMap();
|
||||
|
||||
export function resolveTimeZone(value) {
|
||||
const timeZone = String(value || Intl.DateTimeFormat().resolvedOptions().timeZone || 'UTC').trim();
|
||||
try { new Intl.DateTimeFormat('en-US', { timeZone }).format(0); }
|
||||
catch { throw new Error(`Invalid TZ value: ${timeZone}. Use an IANA time zone such as Europe/Rome.`); }
|
||||
return timeZone;
|
||||
}
|
||||
|
||||
function zonedDateFormatter(timeZone) {
|
||||
const formatter = new Intl.DateTimeFormat('en-US', { timeZone, year:'numeric', month:'2-digit', day:'2-digit' });
|
||||
return epochSeconds => {
|
||||
const parts = Object.fromEntries(formatter.formatToParts(new Date(Number(epochSeconds) * 1000)).map(part => [part.type, part.value]));
|
||||
return `${parts.year}-${parts.month}-${parts.day}`;
|
||||
};
|
||||
}
|
||||
|
||||
function tableColumns(db, table) {
|
||||
return new Set(db.prepare(`PRAGMA table_info(${table})`).all().map(r => r.name));
|
||||
}
|
||||
|
||||
function ensureColumn(db, table, column, definition) {
|
||||
if (!tableColumns(db, table).has(column)) db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`);
|
||||
}
|
||||
|
||||
export function openAppDb(dataDir, { timeZone } = {}) {
|
||||
fs.mkdirSync(dataDir, { recursive: true });
|
||||
const db = new DatabaseSync(path.join(dataDir, 'kovi.sqlite'));
|
||||
const resolvedTimeZone = resolveTimeZone(timeZone);
|
||||
const localDay = zonedDateFormatter(resolvedTimeZone);
|
||||
db.function('kovi_local_day', { deterministic:true }, localDay);
|
||||
dbTimeZones.set(db, resolvedTimeZone);
|
||||
db.exec(`
|
||||
PRAGMA journal_mode=WAL;
|
||||
PRAGMA foreign_keys=ON;
|
||||
PRAGMA busy_timeout=5000;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS books (
|
||||
id TEXT PRIMARY KEY,
|
||||
source_md5 TEXT NOT NULL UNIQUE,
|
||||
title TEXT NOT NULL,
|
||||
authors TEXT,
|
||||
notes INTEGER NOT NULL DEFAULT 0,
|
||||
last_open INTEGER NOT NULL DEFAULT 0,
|
||||
highlights INTEGER NOT NULL DEFAULT 0,
|
||||
pages INTEGER NOT NULL DEFAULT 0,
|
||||
series TEXT,
|
||||
language TEXT,
|
||||
identifiers TEXT,
|
||||
isbn TEXT,
|
||||
total_read_time INTEGER NOT NULL DEFAULT 0,
|
||||
total_read_pages INTEGER NOT NULL DEFAULT 0,
|
||||
koreader_status TEXT CHECK(koreader_status IN ('reading','abandoned','complete')),
|
||||
koreader_status_modified TEXT,
|
||||
read_override INTEGER CHECK(read_override IN (0,1)),
|
||||
read_override_at INTEGER,
|
||||
cover_status TEXT NOT NULL DEFAULT 'pending' CHECK(cover_status IN ('pending','matched','none','error','manual')),
|
||||
cover_path TEXT,
|
||||
cover_source TEXT,
|
||||
cover_retry_mode TEXT,
|
||||
cover_checked_at TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS reading_sessions (
|
||||
id TEXT PRIMARY KEY,
|
||||
fingerprint TEXT NOT NULL UNIQUE,
|
||||
book_id TEXT NOT NULL REFERENCES books(id) ON DELETE CASCADE,
|
||||
device_id TEXT,
|
||||
page INTEGER NOT NULL DEFAULT 0,
|
||||
start_time INTEGER NOT NULL,
|
||||
duration INTEGER NOT NULL,
|
||||
total_pages INTEGER NOT NULL DEFAULT 0,
|
||||
created_at TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_sessions_book_start ON reading_sessions(book_id, start_time DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_sessions_start ON reading_sessions(start_time);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS annotations (
|
||||
id TEXT PRIMARY KEY,
|
||||
fingerprint TEXT NOT NULL,
|
||||
book_id TEXT NOT NULL REFERENCES books(id) ON DELETE CASCADE,
|
||||
device_id TEXT NOT NULL,
|
||||
annotation_type TEXT NOT NULL CHECK(annotation_type IN ('highlight','note','bookmark')),
|
||||
text TEXT,
|
||||
note TEXT,
|
||||
chapter TEXT,
|
||||
page TEXT,
|
||||
pageno INTEGER,
|
||||
total_pages INTEGER,
|
||||
annotation_datetime TEXT,
|
||||
color TEXT,
|
||||
created_at TEXT NOT NULL
|
||||
);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_annotations_source ON annotations(book_id, device_id, fingerprint);
|
||||
CREATE INDEX IF NOT EXISTS idx_annotations_book_date ON annotations(book_id, annotation_datetime DESC);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS imports (
|
||||
id TEXT PRIMARY KEY,
|
||||
source_hash TEXT NOT NULL UNIQUE,
|
||||
kind TEXT NOT NULL,
|
||||
filename TEXT,
|
||||
file_size INTEGER NOT NULL DEFAULT 0,
|
||||
books_seen INTEGER NOT NULL DEFAULT 0,
|
||||
new_books INTEGER NOT NULL DEFAULT 0,
|
||||
sessions_seen INTEGER NOT NULL DEFAULT 0,
|
||||
new_sessions INTEGER NOT NULL DEFAULT 0,
|
||||
warnings_json TEXT NOT NULL DEFAULT '[]',
|
||||
created_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS devices (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT,
|
||||
model TEXT,
|
||||
plugin_version TEXT,
|
||||
token_hash TEXT NOT NULL UNIQUE,
|
||||
created_at TEXT NOT NULL,
|
||||
last_seen_at TEXT,
|
||||
last_sync_at TEXT,
|
||||
sync_cursor INTEGER NOT NULL DEFAULT 0,
|
||||
last_sync_mode TEXT,
|
||||
last_sync_books INTEGER NOT NULL DEFAULT 0,
|
||||
last_sync_stats INTEGER NOT NULL DEFAULT 0,
|
||||
last_sync_new_sessions INTEGER NOT NULL DEFAULT 0,
|
||||
last_sync_annotation_sets INTEGER NOT NULL DEFAULT 0,
|
||||
last_sync_annotations INTEGER NOT NULL DEFAULT 0,
|
||||
revoked_at TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sync_runs (
|
||||
id TEXT PRIMARY KEY,
|
||||
device_id TEXT NOT NULL REFERENCES devices(id) ON DELETE CASCADE,
|
||||
mode TEXT NOT NULL,
|
||||
cursor_before INTEGER NOT NULL DEFAULT 0,
|
||||
cursor_after INTEGER NOT NULL DEFAULT 0,
|
||||
books_seen INTEGER NOT NULL DEFAULT 0,
|
||||
sessions_seen INTEGER NOT NULL DEFAULT 0,
|
||||
new_sessions INTEGER NOT NULL DEFAULT 0,
|
||||
annotation_sets INTEGER NOT NULL DEFAULT 0,
|
||||
annotations_stored INTEGER NOT NULL DEFAULT 0,
|
||||
created_at TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_sync_runs_device_time ON sync_runs(device_id,created_at DESC);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cover_jobs (
|
||||
book_id TEXT PRIMARY KEY REFERENCES books(id) ON DELETE CASCADE,
|
||||
state TEXT NOT NULL CHECK(state IN ('queued','running','done','error')),
|
||||
reason TEXT NOT NULL,
|
||||
force_online INTEGER NOT NULL DEFAULT 0,
|
||||
attempts INTEGER NOT NULL DEFAULT 0,
|
||||
requested_at TEXT NOT NULL,
|
||||
started_at TEXT,
|
||||
finished_at TEXT,
|
||||
last_error TEXT,
|
||||
result_status TEXT,
|
||||
result_source TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_cover_jobs_state_time ON cover_jobs(state,requested_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS book_covers (
|
||||
id TEXT PRIMARY KEY,
|
||||
book_id TEXT NOT NULL REFERENCES books(id) ON DELETE CASCADE,
|
||||
path TEXT NOT NULL,
|
||||
source TEXT,
|
||||
strategy TEXT,
|
||||
score REAL,
|
||||
matched_title TEXT,
|
||||
cover_status TEXT NOT NULL,
|
||||
selected INTEGER NOT NULL DEFAULT 0,
|
||||
created_at TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_book_covers_book_time ON book_covers(book_id,created_at DESC);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_book_covers_selected ON book_covers(book_id) WHERE selected=1;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cover_candidates (
|
||||
id TEXT PRIMARY KEY,
|
||||
book_id TEXT NOT NULL REFERENCES books(id) ON DELETE CASCADE,
|
||||
path TEXT NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
strategy TEXT,
|
||||
score REAL NOT NULL DEFAULT 0,
|
||||
title TEXT,
|
||||
authors TEXT,
|
||||
created_at TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_cover_candidates_book_time ON cover_candidates(book_id,created_at DESC);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS pairing_codes (
|
||||
code_hash TEXT PRIMARY KEY,
|
||||
request_id TEXT,
|
||||
label TEXT,
|
||||
expires_at INTEGER NOT NULL,
|
||||
used_at INTEGER,
|
||||
paired_device_id TEXT
|
||||
);
|
||||
`);
|
||||
|
||||
// Additive in-place schema upgrades for existing databases.
|
||||
ensureColumn(db, 'pairing_codes', 'request_id', 'TEXT');
|
||||
ensureColumn(db, 'pairing_codes', 'paired_device_id', 'TEXT');
|
||||
ensureColumn(db, 'books', 'identifiers', 'TEXT');
|
||||
ensureColumn(db, 'books', 'isbn', 'TEXT');
|
||||
ensureColumn(db, 'books', 'cover_retry_mode', 'TEXT');
|
||||
ensureColumn(db, 'books', 'koreader_status', `TEXT CHECK(koreader_status IN ('reading','abandoned','complete'))`);
|
||||
ensureColumn(db, 'books', 'koreader_status_modified', 'TEXT');
|
||||
ensureColumn(db, 'books', 'read_override', 'INTEGER CHECK(read_override IN (0,1))');
|
||||
ensureColumn(db, 'books', 'read_override_at', 'INTEGER');
|
||||
ensureColumn(db, 'devices', 'last_sync_at', 'TEXT');
|
||||
ensureColumn(db, 'devices', 'sync_cursor', 'INTEGER NOT NULL DEFAULT 0');
|
||||
ensureColumn(db, 'devices', 'last_sync_mode', 'TEXT');
|
||||
ensureColumn(db, 'devices', 'last_sync_books', 'INTEGER NOT NULL DEFAULT 0');
|
||||
ensureColumn(db, 'devices', 'last_sync_stats', 'INTEGER NOT NULL DEFAULT 0');
|
||||
ensureColumn(db, 'devices', 'last_sync_new_sessions', 'INTEGER NOT NULL DEFAULT 0');
|
||||
ensureColumn(db, 'devices', 'last_sync_annotation_sets', 'INTEGER NOT NULL DEFAULT 0');
|
||||
ensureColumn(db, 'devices', 'last_sync_annotations', 'INTEGER NOT NULL DEFAULT 0');
|
||||
db.exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_pairing_request_id ON pairing_codes(request_id) WHERE request_id IS NOT NULL;`);
|
||||
db.exec(`CREATE INDEX IF NOT EXISTS idx_books_isbn ON books(isbn) WHERE isbn IS NOT NULL;`);
|
||||
|
||||
// The same KOReader page-stat row can arrive via manual upload and via a paired
|
||||
// device. Fingerprints differ by ingestion path, which would double-count identical
|
||||
// reading sessions. Normalize once by the event's stable fields, then let SQLite
|
||||
// enforce uniqueness forever.
|
||||
const hasSessionEventIndex = db.prepare(`SELECT 1 ok FROM sqlite_master WHERE type='index' AND name='idx_sessions_event'`).get();
|
||||
if (!hasSessionEventIndex) {
|
||||
db.exec('BEGIN IMMEDIATE');
|
||||
try {
|
||||
db.exec(`DELETE FROM reading_sessions
|
||||
WHERE rowid NOT IN (
|
||||
SELECT MIN(rowid) FROM reading_sessions
|
||||
GROUP BY book_id,page,start_time,duration,total_pages
|
||||
)`);
|
||||
db.exec(`CREATE UNIQUE INDEX idx_sessions_event ON reading_sessions(book_id,page,start_time,duration,total_pages);`);
|
||||
db.exec('COMMIT');
|
||||
} catch (e) { db.exec('ROLLBACK'); throw e; }
|
||||
}
|
||||
|
||||
// Backfill a single selected history entry when no cover history is recorded.
|
||||
const missingHistory=db.prepare(`SELECT id,cover_path,cover_source,cover_status,cover_checked_at,updated_at FROM books WHERE cover_path IS NOT NULL AND NOT EXISTS (SELECT 1 FROM book_covers c WHERE c.book_id=books.id)`).all();
|
||||
const addHistory=db.prepare(`INSERT INTO book_covers(id,book_id,path,source,strategy,score,matched_title,cover_status,selected,created_at) VALUES(?,?,?,?,NULL,NULL,NULL,?,1,?)`);
|
||||
for(const row of missingHistory) addHistory.run(randomId(16),row.id,row.cover_path,row.cover_source,row.cover_status,row.cover_checked_at||row.updated_at||nowIso());
|
||||
return db;
|
||||
}
|
||||
|
||||
function completionView(book) {
|
||||
const total=Number(book.last_total_pages||book.pages||0),page=Number(book.last_page||0);
|
||||
const inferred=total>0 && Math.round(page/total*100)>=100;
|
||||
const isRead=book.read_override == null ? book.koreader_status==='complete'||inferred : Boolean(book.read_override);
|
||||
const readSource=book.read_override != null?'manual':book.koreader_status==='complete'?'koreader':inferred?'progress':null;
|
||||
return {...book,is_read:isRead?1:0,read_source:readSource};
|
||||
}
|
||||
|
||||
export function listBooks(db) {
|
||||
return db.prepare(`
|
||||
SELECT b.*,
|
||||
COALESCE((SELECT MAX(start_time) FROM reading_sessions s WHERE s.book_id=b.id), b.last_open) AS recent_read,
|
||||
COALESCE((SELECT SUM(duration) FROM reading_sessions s WHERE s.book_id=b.id), 0) AS session_read_time,
|
||||
(SELECT page FROM reading_sessions s WHERE s.book_id=b.id ORDER BY start_time DESC LIMIT 1) AS last_page,
|
||||
(SELECT total_pages FROM reading_sessions s WHERE s.book_id=b.id ORDER BY start_time DESC LIMIT 1) AS last_total_pages,
|
||||
(SELECT COUNT(*) FROM (SELECT fingerprint FROM annotations a WHERE a.book_id=b.id AND a.annotation_type IN ('highlight','note') GROUP BY fingerprint)) AS synced_highlights
|
||||
FROM books b
|
||||
ORDER BY recent_read DESC, title COLLATE NOCASE ASC
|
||||
`).all().map(completionView);
|
||||
}
|
||||
|
||||
export function getBook(db, id) {
|
||||
const book = db.prepare(`
|
||||
SELECT b.*,
|
||||
(SELECT page FROM reading_sessions s WHERE s.book_id=b.id ORDER BY start_time DESC LIMIT 1) AS last_page,
|
||||
(SELECT total_pages FROM reading_sessions s WHERE s.book_id=b.id ORDER BY start_time DESC LIMIT 1) AS last_total_pages,
|
||||
(SELECT MAX(start_time) FROM reading_sessions s WHERE s.book_id=b.id) AS recent_read
|
||||
FROM books b WHERE b.id=?
|
||||
`).get(id);
|
||||
if (!book) return null;
|
||||
const annotations = db.prepare(`
|
||||
SELECT fingerprint, annotation_type, text, note, chapter, page, pageno, total_pages, annotation_datetime, color,
|
||||
MAX(created_at) AS synced_at
|
||||
FROM annotations
|
||||
WHERE book_id=? AND annotation_type IN ('highlight','note')
|
||||
GROUP BY fingerprint
|
||||
ORDER BY COALESCE(annotation_datetime,'') DESC, synced_at DESC
|
||||
LIMIT 500
|
||||
`).all(id);
|
||||
return { ...completionView(book), annotations };
|
||||
}
|
||||
|
||||
export function setBookReadOverride(db, id, read, nowEpoch=Math.floor(Date.now()/1000)) {
|
||||
const value=read == null?null:read?1:0;
|
||||
const completedAt=value===1?Math.max(0,Math.floor(Number(nowEpoch)||0)):null;
|
||||
const result=db.prepare(`UPDATE books SET read_override=?,read_override_at=?,updated_at=? WHERE id=?`).run(value,completedAt,nowIso(),id);
|
||||
return result.changes?getBook(db,id):null;
|
||||
}
|
||||
|
||||
function dateKey(date) {
|
||||
return `${date.getUTCFullYear()}-${String(date.getUTCMonth()+1).padStart(2,'0')}-${String(date.getUTCDate()).padStart(2,'0')}`;
|
||||
}
|
||||
|
||||
function parseDateKey(value) {
|
||||
const m=/^(\d{4})-(\d{2})-(\d{2})$/.exec(String(value||''));
|
||||
if(!m) return null;
|
||||
const date=new Date(Date.UTC(Number(m[1]),Number(m[2])-1,Number(m[3]),12,0,0,0));
|
||||
if(date.getUTCFullYear()!==Number(m[1]) || date.getUTCMonth()!==Number(m[2])-1 || date.getUTCDate()!==Number(m[3])) return null;
|
||||
return date;
|
||||
}
|
||||
|
||||
export function dashboardRange({from,to}={}, now=new Date(), timeZone=resolveTimeZone()) {
|
||||
const today=parseDateKey(zonedDateFormatter(resolveTimeZone(timeZone))(now.getTime()/1000));
|
||||
let end=parseDateKey(to) || today;
|
||||
let start=parseDateKey(from);
|
||||
if(!start){start=new Date(end);start.setUTCDate(start.getUTCDate()-364)}
|
||||
if(start>end) [start,end]=[end,start];
|
||||
return {from:dateKey(start),to:dateKey(end)};
|
||||
}
|
||||
|
||||
function queryEpochBounds(start, end) {
|
||||
// IANA UTC offsets are within one day; pad the indexed scan, then filter by local day.
|
||||
const startEpoch=Date.UTC(start.getUTCFullYear(),start.getUTCMonth(),start.getUTCDate()-1)/1000;
|
||||
const endExclusive=Date.UTC(end.getUTCFullYear(),end.getUTCMonth(),end.getUTCDate()+2)/1000;
|
||||
return {startEpoch,endExclusive};
|
||||
}
|
||||
|
||||
export function getDashboard(db, range={}) {
|
||||
const selected=dashboardRange(range,new Date(),dbTimeZones.get(db));
|
||||
const start=parseDateKey(selected.from);
|
||||
const end=parseDateKey(selected.to);
|
||||
const {startEpoch,endExclusive}=queryEpochBounds(start,end);
|
||||
const totals = db.prepare(`SELECT COUNT(*) books, COALESCE(SUM(total_read_time),0) total_read_time, COALESCE(SUM(total_read_pages),0) total_read_pages, COALESCE(SUM(highlights),0) highlights FROM books`).get();
|
||||
const sessions = db.prepare(`SELECT COUNT(*) sessions, COALESCE(SUM(duration),0) session_seconds FROM reading_sessions`).get();
|
||||
const days = db.prepare(`
|
||||
SELECT kovi_local_day(start_time) day, SUM(duration) seconds
|
||||
FROM reading_sessions
|
||||
WHERE start_time >= ? AND start_time < ?
|
||||
GROUP BY day HAVING day >= ? AND day <= ? ORDER BY day
|
||||
`).all(startEpoch,endExclusive,selected.from,selected.to);
|
||||
const rangeSeconds=days.reduce((sum,row)=>sum+Number(row.seconds||0),0);
|
||||
const allTimeReadingSeconds=Number(totals.total_read_time||0)>0 ? Number(totals.total_read_time) : Number(sessions.session_seconds||0);
|
||||
const completionRows=db.prepare(`SELECT read_override,koreader_status,pages,
|
||||
(SELECT page FROM reading_sessions s WHERE s.book_id=books.id ORDER BY start_time DESC LIMIT 1) last_page,
|
||||
(SELECT total_pages FROM reading_sessions s WHERE s.book_id=books.id ORDER BY start_time DESC LIMIT 1) last_total_pages
|
||||
FROM books`).all();
|
||||
const readBooks=completionRows.reduce((count,book)=>count+completionView(book).is_read,0);
|
||||
const rangedReadBooks=db.prepare(`
|
||||
WITH book_dates AS (
|
||||
SELECT b.id,
|
||||
MIN(kovi_local_day(s.start_time)) AS started_on,
|
||||
CASE
|
||||
WHEN b.read_override=1 AND b.read_override_at IS NOT NULL THEN kovi_local_day(b.read_override_at)
|
||||
WHEN b.read_override=1 THEN MAX(kovi_local_day(s.start_time))
|
||||
WHEN b.read_override IS NOT NULL THEN NULL
|
||||
WHEN b.koreader_status='complete' AND b.koreader_status_modified IS NOT NULL THEN b.koreader_status_modified
|
||||
WHEN COALESCE((
|
||||
SELECT ROUND(100.0*c.page/COALESCE(NULLIF(c.total_pages,0),NULLIF(b.pages,0)))
|
||||
FROM reading_sessions c WHERE c.book_id=b.id ORDER BY c.start_time DESC LIMIT 1
|
||||
),0)>=100 THEN (
|
||||
SELECT MIN(kovi_local_day(c.start_time)) FROM reading_sessions c
|
||||
WHERE c.book_id=b.id
|
||||
AND COALESCE(NULLIF(c.total_pages,0),NULLIF(b.pages,0)) IS NOT NULL
|
||||
AND ROUND(100.0*c.page/COALESCE(NULLIF(c.total_pages,0),NULLIF(b.pages,0)))>=100
|
||||
)
|
||||
END AS completed_on
|
||||
FROM books b LEFT JOIN reading_sessions s ON s.book_id=b.id
|
||||
GROUP BY b.id
|
||||
)
|
||||
SELECT COUNT(*) count FROM book_dates
|
||||
WHERE started_on BETWEEN ? AND ? AND completed_on BETWEEN ? AND ? AND completed_on>=started_on
|
||||
`).get(selected.from,selected.to,selected.from,selected.to).count;
|
||||
return {
|
||||
...totals,
|
||||
read_books:readBooks,
|
||||
...sessions,
|
||||
all_time_reading_seconds:allTimeReadingSeconds,
|
||||
reading_time_source:Number(totals.total_read_time||0)>0?'book_totals':'sessions',
|
||||
range:{...selected,seconds:rangeSeconds,reading_days:days.filter(row=>Number(row.seconds||0)>0).length,read_books:Number(rangedReadBooks||0)},
|
||||
days,
|
||||
};
|
||||
}
|
||||
|
||||
export function getCalendar(db, range={}) {
|
||||
const selected=dashboardRange(range,new Date(),dbTimeZones.get(db));
|
||||
const start=parseDateKey(selected.from),end=parseDateKey(selected.to);
|
||||
const {startEpoch,endExclusive}=queryEpochBounds(start,end);
|
||||
const rows=db.prepare(`
|
||||
SELECT kovi_local_day(s.start_time) day,
|
||||
b.id book_id,b.title,b.authors,b.cover_path,b.updated_at,
|
||||
COALESCE(SUM(s.duration),0) seconds,COUNT(*) sessions
|
||||
FROM reading_sessions s
|
||||
JOIN books b ON b.id=s.book_id
|
||||
WHERE s.start_time >= ? AND s.start_time < ?
|
||||
GROUP BY day,b.id HAVING day >= ? AND day <= ?
|
||||
ORDER BY day ASC,seconds DESC,b.title COLLATE NOCASE ASC
|
||||
`).all(startEpoch,endExclusive,selected.from,selected.to);
|
||||
const dayMap=new Map();
|
||||
for(const row of rows){
|
||||
let day=dayMap.get(row.day);
|
||||
if(!day){day={day:row.day,seconds:0,sessions:0,books:[]};dayMap.set(row.day,day)}
|
||||
const seconds=Number(row.seconds||0),sessions=Number(row.sessions||0);day.seconds+=seconds;day.sessions+=sessions;
|
||||
day.books.push({id:row.book_id,title:row.title,authors:row.authors,cover_path:row.cover_path,updated_at:row.updated_at,seconds,sessions});
|
||||
}
|
||||
return {range:selected,days:[...dayMap.values()]};
|
||||
}
|
||||
|
||||
export function createPairingCode(db, label = 'KOReader') {
|
||||
const alphabet = 'ABCDEFGHJKLMNPQRSTUVWXYZ23456789';
|
||||
let code = '';
|
||||
for (let i = 0; i < 8; i++) code += alphabet[randomInt(alphabet.length)];
|
||||
const hash = tokenHash(code);
|
||||
const requestId = randomId(16);
|
||||
const expiresAt = Date.now() + 10 * 60_000;
|
||||
db.prepare(`INSERT INTO pairing_codes(code_hash,request_id,label,expires_at,used_at,paired_device_id) VALUES(?,?,?,?,NULL,NULL)`)
|
||||
.run(hash, requestId, String(label).slice(0,80), expiresAt);
|
||||
return { code, requestId, expiresAt };
|
||||
}
|
||||
|
||||
export function getPairingStatus(db, requestId) {
|
||||
const row = db.prepare(`SELECT request_id,label,expires_at,used_at,paired_device_id FROM pairing_codes WHERE request_id=?`).get(String(requestId || '').slice(0,80));
|
||||
if (!row) return null;
|
||||
if (!row.used_at && row.expires_at < Date.now()) return { status:'expired', expiresAt:row.expires_at };
|
||||
if (!row.used_at) return { status:'pending', expiresAt:row.expires_at };
|
||||
const device = row.paired_device_id ? db.prepare(`SELECT id,name,model,plugin_version,last_seen_at FROM devices WHERE id=?`).get(row.paired_device_id) : null;
|
||||
return { status:'paired', expiresAt:row.expires_at, device:device || null };
|
||||
}
|
||||
|
||||
export function consumePairingCode(db, code, { deviceId, model, version }) {
|
||||
const hash = tokenHash(String(code || '').toUpperCase().replace(/[^A-Z0-9]/g, ''));
|
||||
const row = db.prepare(`SELECT * FROM pairing_codes WHERE code_hash=?`).get(hash);
|
||||
if (!row || row.used_at || row.expires_at < Date.now()) return null;
|
||||
const token = randomToken();
|
||||
const tHash = tokenHash(token);
|
||||
const id = String(deviceId || randomId(8)).slice(0,128);
|
||||
const ts = nowIso();
|
||||
db.exec('BEGIN IMMEDIATE');
|
||||
try {
|
||||
db.prepare(`UPDATE pairing_codes SET used_at=?,paired_device_id=? WHERE code_hash=?`).run(Date.now(), id, hash);
|
||||
db.prepare(`INSERT INTO devices(id,name,model,plugin_version,token_hash,created_at,last_seen_at,revoked_at)
|
||||
VALUES(?,?,?,?,?,?,?,NULL)
|
||||
ON CONFLICT(id) DO UPDATE SET model=excluded.model, plugin_version=excluded.plugin_version, token_hash=excluded.token_hash, last_seen_at=excluded.last_seen_at, revoked_at=NULL`
|
||||
).run(id, row.label, model || null, version || null, tHash, ts, ts);
|
||||
db.exec('COMMIT');
|
||||
} catch (e) { db.exec('ROLLBACK'); throw e; }
|
||||
return { token, deviceId: id, requestId: row.request_id };
|
||||
}
|
||||
|
||||
export function authenticateDevice(db, authorization) {
|
||||
const m = /^Bearer\s+(.+)$/i.exec(authorization || '');
|
||||
if (!m) return null;
|
||||
const hash = tokenHash(m[1]);
|
||||
const row = db.prepare(`SELECT * FROM devices WHERE token_hash=? AND revoked_at IS NULL`).get(hash);
|
||||
if (!row || !safeEqualHex(hash, row.token_hash)) return null;
|
||||
db.prepare(`UPDATE devices SET last_seen_at=? WHERE id=?`).run(nowIso(), row.id);
|
||||
return row;
|
||||
}
|
||||
|
||||
export function updateDevicePluginVersion(db, id, version) {
|
||||
const v=String(version||'').trim().slice(0,40);
|
||||
if(!id || !v) return false;
|
||||
return db.prepare(`UPDATE devices SET plugin_version=?, last_seen_at=? WHERE id=?`).run(v,nowIso(),id).changes>0;
|
||||
}
|
||||
|
||||
export function listDevices(db) {
|
||||
return db.prepare(`SELECT id,name,model,plugin_version,created_at,last_seen_at,last_sync_at,sync_cursor,last_sync_mode,last_sync_books,last_sync_stats,last_sync_new_sessions,last_sync_annotation_sets,last_sync_annotations,revoked_at FROM devices ORDER BY COALESCE(last_seen_at,created_at) DESC`).all();
|
||||
}
|
||||
|
||||
export function recordDeviceSync(db, deviceId, result, {mode='full',cursorBefore=0,cursorAfter=0}={}) {
|
||||
const ts=nowIso();
|
||||
const values=[ts,Math.max(0,Number(cursorAfter)||0),String(mode||'full').slice(0,24),Number(result.booksSeen||0),Number(result.sessionsSeen||0),Number(result.newSessions||0),Number(result.annotationBooks||0),Number(result.annotationsStored||0),deviceId];
|
||||
db.prepare(`UPDATE devices SET last_sync_at=?,sync_cursor=?,last_sync_mode=?,last_sync_books=?,last_sync_stats=?,last_sync_new_sessions=?,last_sync_annotation_sets=?,last_sync_annotations=?,last_seen_at=? WHERE id=?`)
|
||||
.run(...values.slice(0,8),ts,deviceId);
|
||||
db.prepare(`INSERT INTO sync_runs(id,device_id,mode,cursor_before,cursor_after,books_seen,sessions_seen,new_sessions,annotation_sets,annotations_stored,created_at) VALUES(?,?,?,?,?,?,?,?,?,?,?)`)
|
||||
.run(randomId(16),deviceId,String(mode||'full').slice(0,24),Math.max(0,Number(cursorBefore)||0),Math.max(0,Number(cursorAfter)||0),Number(result.booksSeen||0),Number(result.sessionsSeen||0),Number(result.newSessions||0),Number(result.annotationBooks||0),Number(result.annotationsStored||0),ts);
|
||||
return {syncCursor:Math.max(0,Number(cursorAfter)||0),syncAt:ts};
|
||||
}
|
||||
|
||||
export function getStatus(db) {
|
||||
const library=db.prepare(`SELECT COUNT(*) books,COALESCE(SUM(total_read_time),0) reading_seconds,COALESCE(SUM(highlights),0) highlights FROM books`).get();
|
||||
const sessions=db.prepare(`SELECT COUNT(*) sessions FROM reading_sessions`).get();
|
||||
const annotations=db.prepare(`SELECT COUNT(*) annotations FROM annotations WHERE annotation_type IN ('highlight','note')`).get();
|
||||
const covers=db.prepare(`SELECT cover_status status,COUNT(*) count FROM books GROUP BY cover_status`).all();
|
||||
const sources=db.prepare(`SELECT COALESCE(cover_source,'none') source,COUNT(*) count FROM books GROUP BY COALESCE(cover_source,'none') ORDER BY count DESC`).all();
|
||||
const coverJobs=db.prepare(`SELECT state,COUNT(*) count FROM cover_jobs GROUP BY state`).all();
|
||||
const recentImports=db.prepare(`SELECT id,kind,filename,file_size,books_seen,new_books,sessions_seen,new_sessions,warnings_json,created_at FROM imports ORDER BY created_at DESC LIMIT 8`).all().map(r=>({...r,warnings:JSON.parse(r.warnings_json||'[]'),warnings_json:undefined}));
|
||||
const recentSyncs=db.prepare(`SELECT s.*,d.name device_name,d.model device_model FROM sync_runs s LEFT JOIN devices d ON d.id=s.device_id ORDER BY s.created_at DESC LIMIT 12`).all();
|
||||
return {library:{...library,...sessions,...annotations},covers:{statuses:covers,sources,jobs:coverJobs},devices:listDevices(db),recentImports,recentSyncs};
|
||||
}
|
||||
|
||||
export function revokeDevice(db, id) {
|
||||
return db.prepare(`UPDATE devices SET revoked_at=? WHERE id=?`).run(nowIso(), id).changes > 0;
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
import fs from 'node:fs';
|
||||
import fsp from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { pipeline } from 'node:stream/promises';
|
||||
import { createGzip } from 'node:zlib';
|
||||
|
||||
function sqlString(value){return `'${String(value).replaceAll("'","''")}'`}
|
||||
function csv(value){const s=String(value??'');return /[",\r\n]/.test(s)?`"${s.replaceAll('"','""')}"`:s}
|
||||
function md(value){return String(value??'').replace(/\r?\n/g,' ').trim()}
|
||||
function octal(value,width){return Math.max(0,Number(value)||0).toString(8).padStart(width-1,'0').slice(-(width-1))+'\0'}
|
||||
function put(buf,offset,length,value){const src=Buffer.from(String(value));src.copy(buf,offset,0,Math.min(length,src.length))}
|
||||
function tarHeader(name,size,mtime=Math.floor(Date.now()/1000),mode=0o644){
|
||||
const h=Buffer.alloc(512,0);put(h,0,100,name);put(h,100,8,octal(mode,8));put(h,108,8,octal(0,8));put(h,116,8,octal(0,8));put(h,124,12,octal(size,12));put(h,136,12,octal(mtime,12));h.fill(0x20,148,156);h[156]='0'.charCodeAt(0);put(h,257,6,'ustar\0');put(h,263,2,'00');put(h,265,32,'kovi');put(h,297,32,'kovi');let sum=0;for(const b of h)sum+=b;put(h,148,8,sum.toString(8).padStart(6,'0')+'\0 ');return h;
|
||||
}
|
||||
async function writeChunk(stream,chunk){if(!stream.write(chunk)) await new Promise(resolve=>stream.once('drain',resolve))}
|
||||
async function addBuffer(stream,name,buffer){await writeChunk(stream,tarHeader(name,buffer.length));await writeChunk(stream,buffer);const pad=(512-(buffer.length%512))%512;if(pad)await writeChunk(stream,Buffer.alloc(pad))}
|
||||
async function addFile(stream,name,filePath){const st=await fsp.stat(filePath);await writeChunk(stream,tarHeader(name,st.size,Math.floor(st.mtimeMs/1000),st.mode&0o777));for await(const chunk of fs.createReadStream(filePath)) await writeChunk(stream,chunk);const pad=(512-(st.size%512))%512;if(pad)await writeChunk(stream,Buffer.alloc(pad))}
|
||||
|
||||
export async function createBackupArchive(db,dataDir,outputPath,{version='unknown'}={}){
|
||||
const workDir=path.join(dataDir,'uploads');await fsp.mkdir(workDir,{recursive:true});
|
||||
const snapshot=path.join(workDir,`.backup-${Date.now()}-${Math.random().toString(16).slice(2)}.sqlite`);
|
||||
const tarPath=`${outputPath}.tar`;
|
||||
try{
|
||||
db.exec(`VACUUM INTO ${sqlString(snapshot)}`);
|
||||
const manifest={name:'kovi backup',version,generated_at:new Date().toISOString(),database:'kovi.sqlite',covers:[]};
|
||||
const referenced=new Set(db.prepare(`SELECT path FROM book_covers UNION SELECT cover_path path FROM books WHERE cover_path IS NOT NULL`).all().map(r=>r.path).filter(Boolean));
|
||||
const coverFiles=[];
|
||||
for(const webPath of referenced){
|
||||
const relative=String(webPath).replace(/^\/+/, '');
|
||||
if(!relative.startsWith('covers/'))continue;
|
||||
const local=path.join(dataDir,relative);try{const st=await fsp.stat(local);if(st.isFile()){coverFiles.push({webPath,local,relative});manifest.covers.push(webPath)}}catch{}
|
||||
}
|
||||
const tar=fs.createWriteStream(tarPath,{flags:'wx',mode:0o600});
|
||||
await addBuffer(tar,'manifest.json',Buffer.from(JSON.stringify(manifest,null,2)+'\n'));
|
||||
await addFile(tar,'kovi.sqlite',snapshot);
|
||||
for(const file of coverFiles) await addFile(tar,file.relative,file.local);
|
||||
await writeChunk(tar,Buffer.alloc(1024));tar.end();await new Promise((resolve,reject)=>{tar.on('finish',resolve);tar.on('error',reject)});
|
||||
await pipeline(fs.createReadStream(tarPath),createGzip({level:6}),fs.createWriteStream(outputPath,{flags:'wx',mode:0o600}));
|
||||
return {covers:coverFiles.length};
|
||||
} finally {await fsp.rm(snapshot,{force:true}).catch(()=>{});await fsp.rm(tarPath,{force:true}).catch(()=>{});}
|
||||
}
|
||||
|
||||
function humanDuration(seconds){
|
||||
let remaining=Math.max(0,Math.round(Number(seconds)||0));
|
||||
const hours=Math.floor(remaining/3600);remaining%=3600;
|
||||
const minutes=Math.floor(remaining/60);const secs=remaining%60;
|
||||
const parts=[];if(hours)parts.push(`${hours}h`);if(minutes)parts.push(`${minutes}m`);if(secs||!parts.length)parts.push(`${secs}s`);
|
||||
return parts.join(' ');
|
||||
}
|
||||
function humanUnixTime(value){
|
||||
const seconds=Number(value);if(!Number.isFinite(seconds)||seconds<=0)return '';
|
||||
const d=new Date(seconds*1000);if(Number.isNaN(d.getTime()))return '';
|
||||
const pad=n=>String(n).padStart(2,'0');
|
||||
return `${d.getUTCFullYear()}-${pad(d.getUTCMonth()+1)}-${pad(d.getUTCDate())} ${pad(d.getUTCHours())}:${pad(d.getUTCMinutes())}:${pad(d.getUTCSeconds())} UTC`;
|
||||
}
|
||||
|
||||
export function sendBooksCsv(res,db,headers={}){
|
||||
res.writeHead(200,{...headers,'Content-Type':'text/csv; charset=utf-8','Content-Disposition':'attachment; filename="kovi-books.csv"','Cache-Control':'no-store'});
|
||||
res.write('title,authors,series,language,isbn,status,document_progress,reading_time,read_pages,highlights,last_opened,cover_source\r\n');
|
||||
for(const b of db.prepare(`SELECT title,authors,series,language,isbn,total_read_time,total_read_pages,highlights,last_open,cover_source,pages,koreader_status,read_override,(SELECT page FROM reading_sessions s WHERE s.book_id=books.id ORDER BY start_time DESC LIMIT 1) last_page,(SELECT total_pages FROM reading_sessions s WHERE s.book_id=books.id ORDER BY start_time DESC LIMIT 1) last_total_pages FROM books ORDER BY title COLLATE NOCASE`).iterate()){
|
||||
const total=Number(b.last_total_pages||b.pages||0),progress=total>0?Math.min(100,Math.round(Number(b.last_page||0)/total*100)):0;
|
||||
const read=b.read_override==null?b.koreader_status==='complete'||progress===100:Boolean(b.read_override);
|
||||
const status=read?'read':b.read_override===0?'unread':progress?'reading':'unread';
|
||||
res.write([b.title,b.authors,b.series,b.language,b.isbn,status,`${progress}%`,humanDuration(b.total_read_time),b.total_read_pages,b.highlights,humanUnixTime(b.last_open),b.cover_source].map(csv).join(',')+'\r\n');
|
||||
}
|
||||
res.end();
|
||||
}
|
||||
|
||||
export function sendHighlightsMarkdown(res,db,headers={}){
|
||||
res.writeHead(200,{...headers,'Content-Type':'text/markdown; charset=utf-8','Content-Disposition':'attachment; filename="kovi-highlights.md"','Cache-Control':'no-store'});
|
||||
const books=db.prepare(`SELECT id,title,authors FROM books WHERE EXISTS (SELECT 1 FROM annotations a WHERE a.book_id=books.id AND a.annotation_type IN ('highlight','note') AND (a.text IS NOT NULL OR a.note IS NOT NULL)) ORDER BY title COLLATE NOCASE`).all();
|
||||
for(const b of books){res.write(`# ${md(b.title)}\n\n${b.authors?`_${md(b.authors)}_\n\n`:''}`);const anns=db.prepare(`SELECT annotation_type,text,note,chapter,page,pageno,annotation_datetime FROM annotations WHERE book_id=? AND annotation_type IN ('highlight','note') ORDER BY COALESCE(annotation_datetime,created_at)`).all(b.id);for(const a of anns){if(a.text)res.write(`> ${String(a.text).replace(/\r?\n/g,'\n> ')}\n\n`);if(a.note)res.write(`**Note:** ${String(a.note).trim()}\n\n`);const loc=[a.chapter,a.pageno?`page ${a.pageno}`:a.page?`page ${a.page}`:null,a.annotation_datetime].filter(Boolean).map(md).join(' · ');if(loc)res.write(`_${loc}_\n\n`);res.write('---\n\n')}}res.end();
|
||||
}
|
||||
@@ -0,0 +1,278 @@
|
||||
import fs from 'node:fs';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { DatabaseSync } from 'node:sqlite';
|
||||
import { cleanText, asInt, nowIso, randomId, sha256, extractIsbn } from './security.mjs';
|
||||
|
||||
const REQUIRED_BOOK_COLUMNS = ['id','title','authors','notes','last_open','highlights','pages','series','language','md5','total_read_time','total_read_pages'];
|
||||
const REQUIRED_STATS_COLUMNS = ['id_book','page','start_time','duration','total_pages'];
|
||||
|
||||
function columns(db, table) {
|
||||
return db.prepare(`PRAGMA table_info(${table})`).all().map(r => r.name);
|
||||
}
|
||||
function hasAll(actual, needed) { const set = new Set(actual); return needed.every(x => set.has(x)); }
|
||||
|
||||
export function isKoReaderManual(bookOrTitle) {
|
||||
const title = cleanText(typeof bookOrTitle === 'object' ? bookOrTitle?.title : bookOrTitle, 500) || '';
|
||||
const folded = title.toLocaleLowerCase().normalize('NFKD').replace(/[\u0300-\u036f]/g,'').replace(/[^a-z0-9]+/g,' ').trim();
|
||||
if (!folded.includes('koreader')) return false;
|
||||
if (folded === 'koreader') return true;
|
||||
return /\b(user guide|manual|manuale|guida(?: utente)?|guide(?: utilisateur)?|manuel|handbuch|guia(?: del usuario)?|quickstart guide)\b/.test(folded);
|
||||
}
|
||||
|
||||
export function removeExcludedBooks(appDb) {
|
||||
const books = appDb.prepare(`SELECT id,title FROM books`).all();
|
||||
let removed = 0;
|
||||
const del = appDb.prepare(`DELETE FROM books WHERE id=?`);
|
||||
for (const book of books) if (isKoReaderManual(book)) removed += del.run(book.id).changes;
|
||||
return removed;
|
||||
}
|
||||
|
||||
export function validateKoReaderDb(filePath) {
|
||||
const header = Buffer.alloc(16);
|
||||
const fd = fs.openSync(filePath, 'r');
|
||||
fs.readSync(fd, header, 0, 16, 0); fs.closeSync(fd);
|
||||
if (header.toString('ascii',0,16) !== 'SQLite format 3\u0000') throw new Error('This is not a valid SQLite 3 database.');
|
||||
const src = new DatabaseSync(filePath, { readOnly: true });
|
||||
try {
|
||||
const integrity = src.prepare('PRAGMA quick_check').get();
|
||||
if (!integrity || Object.values(integrity)[0] !== 'ok') throw new Error('SQLite integrity check failed.');
|
||||
const tables = new Set(src.prepare(`SELECT name FROM sqlite_master WHERE type='table'`).all().map(r => r.name));
|
||||
if (!tables.has('book') || !tables.has('page_stat_data')) throw new Error('Not a KOReader statistics database: expected book and page_stat_data tables.');
|
||||
const bookCols = columns(src, 'book');
|
||||
const statCols = columns(src, 'page_stat_data');
|
||||
if (!hasAll(bookCols, REQUIRED_BOOK_COLUMNS)) throw new Error(`Unsupported KOReader book schema. Missing: ${REQUIRED_BOOK_COLUMNS.filter(x=>!bookCols.includes(x)).join(', ')}`);
|
||||
if (!hasAll(statCols, REQUIRED_STATS_COLUMNS)) throw new Error(`Unsupported KOReader statistics schema. Missing: ${REQUIRED_STATS_COLUMNS.filter(x=>!statCols.includes(x)).join(', ')}`);
|
||||
return { bookCols, statCols };
|
||||
} finally { src.close(); }
|
||||
}
|
||||
|
||||
export function importKoReaderDb(appDb, filePath, meta = {}) {
|
||||
validateKoReaderDb(filePath);
|
||||
const sourceHash = hashFile(filePath);
|
||||
const prior = appDb.prepare(`SELECT * FROM imports WHERE source_hash=?`).get(sourceHash);
|
||||
if (prior) return { duplicate: true, importId: prior.id, booksSeen: prior.books_seen, newBooks: 0, sessionsSeen: prior.sessions_seen, newSessions: 0, excludedBooks:0, warnings: JSON.parse(prior.warnings_json) };
|
||||
|
||||
const src = new DatabaseSync(filePath, { readOnly: true });
|
||||
const warnings = [];
|
||||
let booksSeen = 0, newBooks = 0, sessionsSeen = 0, newSessions = 0, excludedBooks = 0;
|
||||
const bookIdToMd5 = new Map();
|
||||
const newlyAdded = [];
|
||||
const ts = nowIso();
|
||||
try {
|
||||
const bookCount = Number(src.prepare(`SELECT COUNT(*) n FROM book`).get().n);
|
||||
if (bookCount > 100000) throw new Error('Database contains an unreasonable number of books.');
|
||||
const statCount = Number(src.prepare(`SELECT COUNT(*) n FROM page_stat_data`).get().n);
|
||||
if (statCount > 5_000_000) throw new Error('Database contains an unreasonable number of reading-stat rows.');
|
||||
const books = src.prepare(`SELECT id,title,authors,notes,last_open,highlights,pages,series,language,md5,total_read_time,total_read_pages FROM book`).all();
|
||||
const statsStmt = src.prepare(`SELECT id_book,page,start_time,duration,total_pages FROM page_stat_data`);
|
||||
|
||||
appDb.exec('BEGIN IMMEDIATE');
|
||||
try {
|
||||
const findBook = appDb.prepare(`SELECT id FROM books WHERE source_md5=?`);
|
||||
const insertBook = appDb.prepare(`INSERT INTO books(id,source_md5,title,authors,notes,last_open,highlights,pages,series,language,total_read_time,total_read_pages,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)`);
|
||||
const updateBook = appDb.prepare(`UPDATE books SET title=?,authors=?,notes=?,last_open=?,highlights=?,pages=?,series=?,language=?,total_read_time=?,total_read_pages=?,updated_at=? WHERE source_md5=?`);
|
||||
const sessionInsert = appDb.prepare(`INSERT OR IGNORE INTO reading_sessions(id,fingerprint,book_id,device_id,page,start_time,duration,total_pages,created_at) VALUES(?,?,?,?,?,?,?,?,?)`);
|
||||
|
||||
for (const b of books) {
|
||||
booksSeen++;
|
||||
const md5 = cleanText(b.md5, 128);
|
||||
const title = cleanText(b.title, 500) || 'Untitled';
|
||||
if (isKoReaderManual(title)) { excludedBooks++; continue; }
|
||||
if (!md5) { warnings.push(`Skipped book #${b.id}: missing MD5.`); continue; }
|
||||
bookIdToMd5.set(Number(b.id), md5);
|
||||
const existing = findBook.get(md5);
|
||||
const vals = [title,cleanText(b.authors,500),asInt(b.notes,{max:1e7}),asInt(b.last_open,{max:4e9}),asInt(b.highlights,{max:1e7}),asInt(b.pages,{max:1e7}),cleanText(b.series,500),cleanText(b.language,64),asInt(b.total_read_time,{max:1e12}),asInt(b.total_read_pages,{max:1e9})];
|
||||
if (existing) updateBook.run(...vals, ts, md5);
|
||||
else {
|
||||
const id = randomId(16);
|
||||
insertBook.run(id,md5,...vals,ts,ts);
|
||||
newBooks++; newlyAdded.push(id);
|
||||
}
|
||||
}
|
||||
|
||||
for (const s of statsStmt.iterate()) {
|
||||
sessionsSeen++;
|
||||
const md5 = bookIdToMd5.get(Number(s.id_book));
|
||||
if (!md5) continue;
|
||||
const book = findBook.get(md5);
|
||||
if (!book) continue;
|
||||
const page = asInt(s.page,{max:1e8});
|
||||
const start = asInt(s.start_time,{max:4e9});
|
||||
const duration = asInt(s.duration,{max:7*24*3600});
|
||||
const totalPages = asInt(s.total_pages,{max:1e8});
|
||||
if (!start || duration <= 0) continue;
|
||||
const fp = sha256(`${md5}|${page}|${start}|${duration}|${totalPages}|manual`);
|
||||
const r = sessionInsert.run(randomId(16),fp,book.id,'manual',page,start,duration,totalPages,ts);
|
||||
if (r.changes) newSessions++;
|
||||
}
|
||||
|
||||
const importId = randomId(16);
|
||||
appDb.prepare(`INSERT INTO imports(id,source_hash,kind,filename,file_size,books_seen,new_books,sessions_seen,new_sessions,warnings_json,created_at) VALUES(?,?,?,?,?,?,?,?,?,?,?)`)
|
||||
.run(importId,sourceHash,'sqlite',cleanText(meta.filename,255),Number(meta.fileSize||0),booksSeen,newBooks,sessionsSeen,newSessions,JSON.stringify(warnings.slice(0,100)),ts);
|
||||
appDb.exec('COMMIT');
|
||||
return { duplicate:false, importId, sourceHash, booksSeen,newBooks,sessionsSeen,newSessions,excludedBooks,warnings:warnings.slice(0,100),newBookIds:newlyAdded };
|
||||
} catch (e) { appDb.exec('ROLLBACK'); throw e; }
|
||||
} finally { src.close(); }
|
||||
}
|
||||
|
||||
function annotationType(a) {
|
||||
if (cleanText(a?.note, 20000)) return 'note';
|
||||
if (cleanText(a?.text, 20000) || a?.drawer || a?.highlighted) return 'highlight';
|
||||
return 'bookmark';
|
||||
}
|
||||
|
||||
function cleanDateKey(value) {
|
||||
const key=cleanText(value,40);const match=/^(\d{4})-(\d{2})-(\d{2})$/.exec(key||'');
|
||||
if(!match) return null;
|
||||
const date=new Date(Date.UTC(Number(match[1]),Number(match[2])-1,Number(match[3])));
|
||||
return date.getUTCFullYear()===Number(match[1])&&date.getUTCMonth()===Number(match[2])-1&&date.getUTCDate()===Number(match[3])?key:null;
|
||||
}
|
||||
|
||||
function importAnnotationSets(appDb, sets, device, findBook, ts) {
|
||||
if (!Array.isArray(sets)) return { annotationBooks:0, annotationsSeen:0, annotationsStored:0, metadataEnriched:0, coverRefreshIds:[] };
|
||||
if (sets.length > 10000) throw new Error('Plugin annotation payload contains too many books.');
|
||||
let annotationsSeen = 0, annotationsStored = 0, annotationBooks = 0, metadataEnriched = 0;
|
||||
const coverRefreshIds = [];
|
||||
const del = appDb.prepare(`DELETE FROM annotations WHERE book_id=? AND device_id=?`);
|
||||
const insert = appDb.prepare(`INSERT OR IGNORE INTO annotations(id,fingerprint,book_id,device_id,annotation_type,text,note,chapter,page,pageno,total_pages,annotation_datetime,color,created_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)`);
|
||||
const updateMetadata = appDb.prepare(`UPDATE books SET identifiers=COALESCE(?,identifiers),isbn=COALESCE(?,isbn),cover_status=?,cover_checked_at=CASE WHEN ? THEN NULL ELSE cover_checked_at END,updated_at=? WHERE id=?`);
|
||||
const updateStatus = appDb.prepare(`UPDATE books SET koreader_status=?,koreader_status_modified=?,updated_at=? WHERE id=? AND (koreader_status_modified IS NULL OR (? IS NOT NULL AND koreader_status_modified<=?))`);
|
||||
|
||||
for (const set of sets) {
|
||||
const md5 = cleanText(set?.book_md5, 128);
|
||||
if (!md5) continue;
|
||||
const book = findBook.get(md5);
|
||||
if (!book) continue;
|
||||
|
||||
const readingStatus=cleanText(set?.reading_status,40);
|
||||
if (['reading','abandoned','complete'].includes(readingStatus)) {
|
||||
const statusModified=cleanDateKey(set?.reading_status_modified);
|
||||
updateStatus.run(readingStatus,statusModified,ts,book.id,statusModified,statusModified);
|
||||
}
|
||||
|
||||
// KOReader stores document metadata, including dc:identifier/ISBN, in each
|
||||
// book's doc_props sidecar. Enrich the canonical book record when the plugin
|
||||
// sends it so translated titles can use deterministic ISBN cover lookup.
|
||||
const identifiers = cleanText(set?.identifiers ?? set?.metadata?.identifiers, 4000);
|
||||
const isbn = extractIsbn(set?.isbn ?? identifiers);
|
||||
if ((identifiers && identifiers !== book.identifiers) || (isbn && isbn !== book.isbn)) {
|
||||
const learnedIsbn = Boolean(isbn && !book.isbn);
|
||||
const shouldRetryCover = learnedIsbn && ['none','error'].includes(book.cover_status);
|
||||
updateMetadata.run(
|
||||
identifiers || null,
|
||||
isbn || null,
|
||||
shouldRetryCover ? 'pending' : book.cover_status,
|
||||
shouldRetryCover ? 1 : 0,
|
||||
ts,
|
||||
book.id,
|
||||
);
|
||||
metadataEnriched++;
|
||||
if (shouldRetryCover) coverRefreshIds.push(book.id);
|
||||
}
|
||||
|
||||
const annotations = Array.isArray(set.annotations) ? set.annotations : [];
|
||||
if (annotations.length > 5000) throw new Error('A book contains an unreasonable number of annotations.');
|
||||
annotationBooks++;
|
||||
del.run(book.id, device.id);
|
||||
for (const a of annotations) {
|
||||
annotationsSeen++;
|
||||
const text = cleanText(a?.text ?? a?.notes, 20000);
|
||||
const note = cleanText(a?.note, 20000);
|
||||
const chapter = cleanText(a?.chapter, 2000);
|
||||
const page = cleanText(a?.page, 1000);
|
||||
const pageno = a?.pageno == null ? null : asInt(a.pageno,{max:1e8});
|
||||
const totalPages = a?.total_pages == null ? null : asInt(a.total_pages,{max:1e8});
|
||||
const datetime = cleanText(a?.datetime, 100);
|
||||
const color = cleanText(a?.color, 80);
|
||||
const type = annotationType(a);
|
||||
if (!text && !note) continue;
|
||||
const fp = sha256(`${datetime||''}|${pageno??''}|${page||''}|${chapter||''}|${text||''}|${note||''}|${type}`);
|
||||
annotationsStored += insert.run(randomId(16),fp,book.id,device.id,type,text,note,chapter,page,pageno,totalPages,datetime,color,ts).changes;
|
||||
}
|
||||
}
|
||||
return { annotationBooks, annotationsSeen, annotationsStored, metadataEnriched, coverRefreshIds:[...new Set(coverRefreshIds)] };
|
||||
}
|
||||
|
||||
export function importPluginPayload(appDb, payload, device) {
|
||||
const books = Array.isArray(payload?.books) ? payload.books : [];
|
||||
const stats = Array.isArray(payload?.stats) ? payload.stats : [];
|
||||
const annotationSets = Array.isArray(payload?.annotation_sets) ? payload.annotation_sets : [];
|
||||
if (books.length > 10000 || stats.length > 500000) throw new Error('Plugin payload is too large.');
|
||||
const syncCursorBefore=Math.max(0,asInt(payload?.sync_cursor,{max:4e9}));
|
||||
const syncMode=syncCursorBefore>0?'incremental':'full';
|
||||
let syncCursorAfter=syncCursorBefore;
|
||||
const ts = nowIso();
|
||||
let newBooks=0,newSessions=0,excludedBooks=0;
|
||||
const newBookIds=[];
|
||||
let annotationResult={annotationBooks:0,annotationsSeen:0,annotationsStored:0,metadataEnriched:0,coverRefreshIds:[]};
|
||||
appDb.exec('BEGIN IMMEDIATE');
|
||||
try {
|
||||
const findBook=appDb.prepare(`SELECT id,isbn,identifiers,cover_status,cover_source FROM books WHERE source_md5=?`);
|
||||
const insert=appDb.prepare(`INSERT INTO books(id,source_md5,title,authors,notes,last_open,highlights,pages,series,language,identifiers,isbn,total_read_time,total_read_pages,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`);
|
||||
const update=appDb.prepare(`UPDATE books SET title=?,authors=?,notes=?,last_open=?,highlights=?,pages=?,series=?,language=?,identifiers=COALESCE(?,identifiers),isbn=COALESCE(?,isbn),total_read_time=?,total_read_pages=?,updated_at=? WHERE source_md5=?`);
|
||||
for(const b of books){
|
||||
const title=cleanText(b?.title,500)||'Untitled';
|
||||
if(isKoReaderManual(title)){excludedBooks++;continue;}
|
||||
const md5=cleanText(b?.md5,128); if(!md5) continue;
|
||||
const identifiers=cleanText(b?.identifiers,4000);
|
||||
const isbn=extractIsbn(b?.isbn ?? identifiers);
|
||||
const vals=[title,cleanText(b?.authors,500),asInt(b?.notes,{max:1e7}),asInt(b?.last_open,{max:4e9}),asInt(b?.highlights,{max:1e7}),asInt(b?.pages,{max:1e7}),cleanText(b?.series,500),cleanText(b?.language,64),identifiers,isbn,asInt(b?.total_read_time,{max:1e12}),asInt(b?.total_read_pages,{max:1e9})];
|
||||
const ex=findBook.get(md5);
|
||||
if(ex) {
|
||||
update.run(...vals,ts,md5);
|
||||
if(isbn && !ex.isbn && ['none','error'].includes(ex.cover_status)) {
|
||||
appDb.prepare(`UPDATE books SET cover_status='pending',cover_checked_at=NULL WHERE id=?`).run(ex.id);
|
||||
annotationResult.coverRefreshIds.push(ex.id);
|
||||
}
|
||||
} else {
|
||||
const id=randomId(16); insert.run(id,md5,...vals,ts,ts); newBooks++; newBookIds.push(id);
|
||||
}
|
||||
}
|
||||
const insS=appDb.prepare(`INSERT OR IGNORE INTO reading_sessions(id,fingerprint,book_id,device_id,page,start_time,duration,total_pages,created_at) VALUES(?,?,?,?,?,?,?,?,?)`);
|
||||
for(const s of stats){
|
||||
const md5=cleanText(s?.book_md5,128); if(!md5) continue;
|
||||
const book=findBook.get(md5); if(!book) continue;
|
||||
const page=asInt(s.page,{max:1e8}), start=asInt(s.start_time,{max:4e9}), duration=asInt(s.duration,{max:7*24*3600}), totalPages=asInt(s.total_pages,{max:1e8});
|
||||
if(start>syncCursorAfter) syncCursorAfter=start;
|
||||
if(!start||duration<=0) continue;
|
||||
const fp=sha256(`${md5}|${page}|${start}|${duration}|${totalPages}|${device.id}`);
|
||||
if(insS.run(randomId(16),fp,book.id,device.id,page,start,duration,totalPages,ts).changes) newSessions++;
|
||||
}
|
||||
const ann = importAnnotationSets(appDb,annotationSets,device,findBook,ts);
|
||||
annotationResult = {
|
||||
...ann,
|
||||
coverRefreshIds:[...new Set([...(annotationResult.coverRefreshIds||[]),...(ann.coverRefreshIds||[])])],
|
||||
};
|
||||
appDb.exec('COMMIT');
|
||||
} catch(e){appDb.exec('ROLLBACK');throw e;}
|
||||
|
||||
// Ask the current reader for embedded covers only for books that still need one.
|
||||
// Cap the list so a single sync remains responsive on slower e-ink devices.
|
||||
const coverRequests=[];
|
||||
const seen=new Set();
|
||||
const coverState=appDb.prepare(`SELECT source_md5,cover_status,cover_source FROM books WHERE source_md5=?`);
|
||||
for(const b of books){
|
||||
const md5=cleanText(b?.md5,128); if(!md5 || seen.has(md5)) continue;
|
||||
seen.add(md5);
|
||||
const row=coverState.get(md5);
|
||||
if(row && ['pending','none','error'].includes(row.cover_status) && row.cover_source!=='koreader-embedded') coverRequests.push(md5);
|
||||
if(coverRequests.length>=20) break;
|
||||
}
|
||||
|
||||
return {
|
||||
booksSeen:books.length,newBooks,sessionsSeen:stats.length,newSessions,newBookIds,excludedBooks,
|
||||
...annotationResult,
|
||||
coverRefreshIds:[...new Set(annotationResult.coverRefreshIds||[])],
|
||||
coverRequests,
|
||||
syncMode,syncCursorBefore,syncCursorAfter,
|
||||
};
|
||||
}
|
||||
|
||||
function hashFile(filePath) {
|
||||
const h = createHash('sha256');
|
||||
const fd = fs.openSync(filePath, 'r');
|
||||
const b = Buffer.allocUnsafe(1024*1024);
|
||||
try { let n; while ((n=fs.readSync(fd,b,0,b.length,null))>0) h.update(b.subarray(0,n)); }
|
||||
finally { fs.closeSync(fd); }
|
||||
return h.digest('hex');
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
function cleanValue(value) {
|
||||
if (value === undefined || value === null || value === '') return undefined;
|
||||
if (value instanceof Error) return value.message;
|
||||
if (typeof value === 'string') return value.replace(/[\r\n\t]+/g, ' ').slice(0, 500);
|
||||
if (typeof value === 'number' || typeof value === 'boolean') return value;
|
||||
if (Array.isArray(value)) return value.slice(0, 20).map(cleanValue);
|
||||
if (typeof value === 'object') {
|
||||
const out = {};
|
||||
for (const [key, item] of Object.entries(value)) {
|
||||
if (/token|authorization|code$/i.test(key)) continue;
|
||||
const cleaned = cleanValue(item);
|
||||
if (cleaned !== undefined) out[key] = cleaned;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
return String(value).slice(0, 500);
|
||||
}
|
||||
|
||||
export function logEvent(level, event, message, fields = {}) {
|
||||
const normalized = String(level || 'info').toLowerCase();
|
||||
const meta = cleanValue(fields) || {};
|
||||
const suffix = Object.keys(meta).length ? ` ${JSON.stringify(meta)}` : '';
|
||||
const line = `[${new Date().toISOString()}] [kovi] ${normalized.toUpperCase()} ${event} — ${message}${suffix}`;
|
||||
if (normalized === 'error') console.error(line);
|
||||
else if (normalized === 'warn' || normalized === 'warning') console.warn(line);
|
||||
else console.log(line);
|
||||
}
|
||||
|
||||
export const info = (event, message, fields) => logEvent('info', event, message, fields);
|
||||
export const warn = (event, message, fields) => logEvent('warn', event, message, fields);
|
||||
export const error = (event, message, fields) => logEvent('error', event, message, fields);
|
||||
@@ -0,0 +1,68 @@
|
||||
import { createHash, randomBytes, timingSafeEqual } from 'node:crypto';
|
||||
|
||||
export const nowIso = () => new Date().toISOString();
|
||||
export const randomId = (bytes = 16) => randomBytes(bytes).toString('hex');
|
||||
export const randomToken = () => `kv_${randomBytes(24).toString('base64url')}`;
|
||||
export const tokenHash = (token) => createHash('sha256').update(token).digest('hex');
|
||||
export const sha256 = (input) => createHash('sha256').update(input).digest('hex');
|
||||
|
||||
export function safeEqualHex(a, b) {
|
||||
if (!a || !b || a.length !== b.length) return false;
|
||||
return timingSafeEqual(Buffer.from(a, 'hex'), Buffer.from(b, 'hex'));
|
||||
}
|
||||
|
||||
export function cleanText(value, max = 500) {
|
||||
if (value == null) return null;
|
||||
const s = String(value).replace(/[\u0000-\u001f\u007f]/g, ' ').replace(/\s+/g, ' ').trim();
|
||||
return s.slice(0, max) || null;
|
||||
}
|
||||
|
||||
export function asInt(value, { min = 0, max = Number.MAX_SAFE_INTEGER, fallback = 0 } = {}) {
|
||||
const n = Number(value);
|
||||
if (!Number.isFinite(n)) return fallback;
|
||||
const i = Math.trunc(n);
|
||||
return Math.max(min, Math.min(max, i));
|
||||
}
|
||||
|
||||
function validIsbn10(value) {
|
||||
if (!/^\d{9}[\dX]$/.test(value)) return false;
|
||||
let sum = 0;
|
||||
for (let i = 0; i < 10; i++) {
|
||||
const d = value[i] === 'X' ? 10 : Number(value[i]);
|
||||
sum += d * (10 - i);
|
||||
}
|
||||
return sum % 11 === 0;
|
||||
}
|
||||
|
||||
function validIsbn13(value) {
|
||||
if (!/^97[89]\d{10}$/.test(value)) return false;
|
||||
let sum = 0;
|
||||
for (let i = 0; i < 12; i++) sum += Number(value[i]) * (i % 2 ? 3 : 1);
|
||||
return (10 - (sum % 10)) % 10 === Number(value[12]);
|
||||
}
|
||||
|
||||
/** Extract a validated ISBN from EPUB/KOReader identifier metadata.
|
||||
* KOReader's doc_props.identifiers is commonly a newline-separated string such as
|
||||
* `isbn:978...`, `urn:isbn:...`, `calibre:...`, `uuid:...`.
|
||||
*/
|
||||
export function extractIsbn(value) {
|
||||
if (!value) return null;
|
||||
const raw = String(value).toUpperCase();
|
||||
const candidates = raw.match(/(?:97[89][\d\s-]{9,20}\d|\d[\d\s-]{7,16}[\dX])/g) || [];
|
||||
const cleaned = [...new Set(candidates.map(x => x.replace(/[^0-9X]/g, '')))];
|
||||
const isbn13 = cleaned.find(x => x.length === 13 && validIsbn13(x));
|
||||
if (isbn13) return isbn13;
|
||||
return cleaned.find(x => x.length === 10 && validIsbn10(x)) || null;
|
||||
}
|
||||
|
||||
export function json(res, status, body, extraHeaders = {}) {
|
||||
const payload = Buffer.from(JSON.stringify(body));
|
||||
res.writeHead(status, {
|
||||
'Content-Type': 'application/json; charset=utf-8',
|
||||
'Content-Length': payload.length,
|
||||
'Cache-Control': 'no-store',
|
||||
'X-Content-Type-Options': 'nosniff',
|
||||
...extraHeaders,
|
||||
});
|
||||
res.end(payload);
|
||||
}
|
||||
Reference in New Issue
Block a user