mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-11 22:02:16 +00:00
feat(documents): search and import from Paperless-ngx and Papra on request
A person stores one connection per system (base address, Papra organization id, API token encrypted at rest) and can search it by document name with tag and date filters, then import picked documents one per request through the same ingest path as an upload: source-key answer before any download, content dedup, deleted stays deleted, quota, provenance, and the person's own AI reading setting. An import can link the document to the visit, condition or vaccination it was picked for. The operator lists the instances in DOCUMENT_SOURCE_ORIGINS; unset, the feature is off. Only listed origins are contacted, through safeFetch with the origin pinned, redirects refused, timeouts and byte caps. The routes are cookie-only and owner-only. The token is registered for rotation, removed by Delete all data, and not carried in backups.
This commit is contained in:
1 parent
89b18e001e
commit
fdc60f69c2
26 files changed
+2305
-1
No files matched your search
@@ -0,0 +1,294 @@
|
||||
/**
|
||||
* Document picker (#1038): import one picked document from a connected
|
||||
* archive into the vault.
|
||||
*
|
||||
* One document per request, so the sheet can show each one's progress and
|
||||
* result, and no single request holds more than one file in memory. The path:
|
||||
*
|
||||
* 1. the link target, when the picker was opened from a record, must be one
|
||||
* of the person's own live records (404 before anything leaves the
|
||||
* server);
|
||||
* 2. the source key answers first, without a download: a document already
|
||||
* imported (or already holding this key as an alias) is a `duplicate`,
|
||||
* and one the person deleted here is `deleted` — nothing is fetched and
|
||||
* nothing is stored, exactly as the token upload answers the same key;
|
||||
* 3. only then the vault's own upload allowance (60 an hour, shared with
|
||||
* uploads from the web and the phone) and an upload slot are taken, the
|
||||
* document's details and its original file are fetched (bounded by the
|
||||
* vault's per-file cap), and the file goes through `ingestDocument`, the
|
||||
* same function the upload route stores through: type check, content
|
||||
* dedup, quota, provenance (`sourceSystem` / `sourceId`), the background
|
||||
* jobs;
|
||||
* 4. the document is linked to the record through `src/lib/links/`.
|
||||
*
|
||||
* AI reading follows the person's own setting, unlike the token upload's
|
||||
* default of holding it back: the person picked this document by hand, as
|
||||
* deliberately as an upload. The hourly allowance and the daily AI budget
|
||||
* bound it either way.
|
||||
*
|
||||
* Cookie-only and owner-only; see `admitDocumentSourceCaller`.
|
||||
*/
|
||||
import { Buffer } from "node:buffer";
|
||||
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
|
||||
import { apiHandler, requireAuth } from "@/lib/api-handler";
|
||||
import {
|
||||
apiError,
|
||||
apiValidationError,
|
||||
getClientIp,
|
||||
safeJson,
|
||||
sanitiseZodIssues,
|
||||
} from "@/lib/api-response";
|
||||
import { prisma } from "@/lib/db";
|
||||
import {
|
||||
ingestDocument,
|
||||
personalUploadBucket,
|
||||
UPLOAD_LIMIT_PER_HOUR,
|
||||
UPLOAD_WINDOW_MS,
|
||||
} from "@/lib/documents/ingest";
|
||||
import { findSourceKey, type SourceKeyMatch } from "@/lib/documents/source-key";
|
||||
import { clientFor, loadConnection } from "@/lib/documents/sources/connections";
|
||||
import {
|
||||
admitDocumentSourceCaller,
|
||||
responseForSourceFailure,
|
||||
sourceErrorResponse,
|
||||
systemParam,
|
||||
} from "@/lib/documents/sources/route-support";
|
||||
import type {
|
||||
DocumentImportOutcome,
|
||||
DocumentPickerLinkKind,
|
||||
DocumentSourceImportDto,
|
||||
} from "@/lib/documents/sources/types";
|
||||
import {
|
||||
acquireDocumentUploadSlot,
|
||||
resolveDocumentLimits,
|
||||
} from "@/lib/documents/upload-policy";
|
||||
import { linkTargets } from "@/lib/links";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import { checkRateLimit, rateLimitHeaders } from "@/lib/rate-limit";
|
||||
import { documentSourceImportSchema } from "@/lib/validations/document-sources";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
type RouteParams = { params: Promise<{ system: string }> };
|
||||
|
||||
interface LinkTarget {
|
||||
kind: DocumentPickerLinkKind;
|
||||
id: string;
|
||||
}
|
||||
|
||||
/** Is the record the picker was opened from one of the person's live ones? */
|
||||
async function ownsLinkTarget(
|
||||
userId: string,
|
||||
link: LinkTarget,
|
||||
): Promise<boolean> {
|
||||
const where = { id: link.id, userId, deletedAt: null };
|
||||
const select = { id: true } as const;
|
||||
switch (link.kind) {
|
||||
case "conditionEpisode":
|
||||
return (
|
||||
(await prisma.illnessEpisode.findFirst({ where, select })) !== null
|
||||
);
|
||||
case "encounter":
|
||||
return (await prisma.encounter.findFirst({ where, select })) !== null;
|
||||
case "vaccination":
|
||||
return (
|
||||
(await prisma.vaccinationRecord.findFirst({ where, select })) !== null
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/** File the document against the record, from the record's side. */
|
||||
async function linkDocument(
|
||||
userId: string,
|
||||
link: LinkTarget | undefined,
|
||||
documentId: string,
|
||||
): Promise<boolean> {
|
||||
if (!link) return false;
|
||||
const result = await prisma.$transaction((tx) =>
|
||||
linkTargets(tx, {
|
||||
userId,
|
||||
sourceKind: link.kind,
|
||||
sourceId: link.id,
|
||||
targetKind: "document",
|
||||
targetIds: [documentId],
|
||||
}),
|
||||
);
|
||||
return !result.unknownSource && result.unknownTargetIds.length === 0;
|
||||
}
|
||||
|
||||
function respond(
|
||||
system: string,
|
||||
outcome: DocumentImportOutcome,
|
||||
documentId: string | null,
|
||||
linked: boolean,
|
||||
): NextResponse {
|
||||
annotate({
|
||||
action: { name: "documents.sources.import" },
|
||||
meta: { system, outcome, linked, documentId },
|
||||
});
|
||||
const data: DocumentSourceImportDto = { outcome, documentId, linked };
|
||||
return NextResponse.json(
|
||||
{ data, error: null },
|
||||
{ status: outcome === "imported" ? 201 : 200 },
|
||||
);
|
||||
}
|
||||
|
||||
/** A key the vault already holds: link a live one, never revive a deleted one. */
|
||||
async function answerHeldKey(
|
||||
system: string,
|
||||
userId: string,
|
||||
link: LinkTarget | undefined,
|
||||
match: SourceKeyMatch,
|
||||
): Promise<NextResponse> {
|
||||
if (match.state === "deleted") {
|
||||
return respond(system, "deleted", match.id, false);
|
||||
}
|
||||
const linked = await linkDocument(userId, link, match.document.id);
|
||||
return respond(system, "duplicate", match.document.id, linked);
|
||||
}
|
||||
|
||||
export const POST = apiHandler(
|
||||
async (request: NextRequest, { params }: RouteParams) => {
|
||||
const auth = await requireAuth();
|
||||
const refused = await admitDocumentSourceCaller(auth);
|
||||
if (refused) return refused;
|
||||
const system = await systemParam(params);
|
||||
if (system instanceof Response) return system;
|
||||
const userId = auth.user.id;
|
||||
|
||||
const { data: body, error: jsonError } = await safeJson(request, {
|
||||
maxBytes: 4 * 1024,
|
||||
});
|
||||
if (jsonError) return jsonError;
|
||||
const parsed = documentSourceImportSchema.safeParse(body);
|
||||
if (!parsed.success) {
|
||||
return apiValidationError(
|
||||
"Invalid import request",
|
||||
sanitiseZodIssues(parsed.error.issues),
|
||||
422,
|
||||
);
|
||||
}
|
||||
const { sourceId, kind, link } = parsed.data;
|
||||
|
||||
const row = await loadConnection(userId, system);
|
||||
if (!row) return sourceErrorResponse("notConnected");
|
||||
let client;
|
||||
try {
|
||||
client = clientFor(row);
|
||||
} catch (err) {
|
||||
return responseForSourceFailure(err);
|
||||
}
|
||||
|
||||
if (link && !(await ownsLinkTarget(userId, link))) {
|
||||
return sourceErrorResponse("linkTargetNotFound");
|
||||
}
|
||||
|
||||
// The source key answers before anything is downloaded or charged.
|
||||
const held = await findSourceKey(userId, system, sourceId);
|
||||
if (held) return answerHeldKey(system, userId, link, held);
|
||||
|
||||
const rl = await checkRateLimit(
|
||||
personalUploadBucket(userId),
|
||||
UPLOAD_LIMIT_PER_HOUR,
|
||||
UPLOAD_WINDOW_MS,
|
||||
);
|
||||
if (!rl.allowed) {
|
||||
return apiError("Too many uploads. Try again later.", 429, {
|
||||
errorCode: "documents.inbound.rateLimited",
|
||||
headers: rateLimitHeaders(rl),
|
||||
});
|
||||
}
|
||||
|
||||
const release = acquireDocumentUploadSlot(userId);
|
||||
if (!release) {
|
||||
return apiError(
|
||||
"Too many uploads are already in progress. Try again shortly.",
|
||||
429,
|
||||
{
|
||||
errorCode: "documents.inbound.uploadBusy",
|
||||
reason: "uploadBusy",
|
||||
headers: { "Retry-After": "1" },
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const limits = await resolveDocumentLimits(userId);
|
||||
let meta;
|
||||
let file;
|
||||
try {
|
||||
meta = await client.document(sourceId);
|
||||
file = await client.download(sourceId, limits.maxFileBytes);
|
||||
} catch (err) {
|
||||
return responseForSourceFailure(err);
|
||||
}
|
||||
|
||||
const result = await ingestDocument({
|
||||
userId,
|
||||
scoped: false,
|
||||
ipAddress: getClientIp(request),
|
||||
bytes: Buffer.from(file.bytes),
|
||||
filename: meta.filename ?? file.filename,
|
||||
title: meta.title,
|
||||
kind: kind ?? null,
|
||||
documentDate: meta.date,
|
||||
episodeIds: [],
|
||||
encounterIds: [],
|
||||
sourceSystem: system,
|
||||
sourceId,
|
||||
aiDeferred: false,
|
||||
sourceKeyChecked: true,
|
||||
limits,
|
||||
});
|
||||
|
||||
switch (result.kind) {
|
||||
case "sourceKey":
|
||||
return answerHeldKey(system, userId, link, result.match);
|
||||
case "duplicate": {
|
||||
// The same bytes were already in the vault (an upload, or another
|
||||
// archive); the key is now remembered for that document.
|
||||
const linked = await linkDocument(userId, link, result.document.id);
|
||||
return respond(system, "duplicate", result.document.id, linked);
|
||||
}
|
||||
case "stored": {
|
||||
const linked = await linkDocument(userId, link, result.document.id);
|
||||
return respond(system, "imported", result.document.id, linked);
|
||||
}
|
||||
case "tooLarge":
|
||||
return apiError("File is too large.", 413, {
|
||||
errorCode: "documents.inbound.fileTooLarge",
|
||||
reason: "fileTooLarge",
|
||||
maxFileBytes: result.maxFileBytes,
|
||||
});
|
||||
case "empty":
|
||||
case "unsupportedType":
|
||||
return apiError("This file type is not supported.", 415, {
|
||||
errorCode: "documents.inbound.fileType",
|
||||
reason: "unsupportedType",
|
||||
});
|
||||
case "quotaExceeded":
|
||||
return apiError("Storage quota exceeded.", 413, {
|
||||
errorCode: "documents.inbound.quotaExceeded",
|
||||
reason: "quotaExceeded",
|
||||
quotaBytes: result.quotaBytes,
|
||||
usedBytes: result.usedBytes,
|
||||
});
|
||||
case "aliasLimit":
|
||||
return apiError(
|
||||
"This file is already stored under too many source ids.",
|
||||
409,
|
||||
{ errorCode: "documents.inbound.sourceAliasLimit" },
|
||||
);
|
||||
case "episodeNotFound":
|
||||
case "encounterNotFound":
|
||||
// No pre-links are passed on this path; unreachable, but a result
|
||||
// the switch does not name must not fall through silently.
|
||||
return sourceErrorResponse("linkTargetNotFound");
|
||||
}
|
||||
} finally {
|
||||
release();
|
||||
}
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,191 @@
|
||||
/**
|
||||
* Document picker (#1038): save or remove the person's connection to one
|
||||
* document archive (`paperless` or `papra`).
|
||||
*
|
||||
* PUT validates the base address against the operator's list
|
||||
* (`DOCUMENT_SOURCE_ORIGINS`), tests the connection live before storing
|
||||
* anything (a wrong token or an unreachable host is reported here, not on the
|
||||
* first search), then stores the token encrypted. A PUT without a token keeps
|
||||
* the stored one, so the address or the Papra organization can change without
|
||||
* typing the key again; the first save must carry it. The response is the
|
||||
* connection without its token, like every read of it.
|
||||
*
|
||||
* DELETE removes the row, and with it the token.
|
||||
*
|
||||
* Cookie-only and owner-only; see `admitDocumentSourceCaller`.
|
||||
*/
|
||||
import { NextRequest } from "next/server";
|
||||
|
||||
import { apiHandler, requireAuth } from "@/lib/api-handler";
|
||||
import {
|
||||
apiError,
|
||||
apiSuccess,
|
||||
apiValidationError,
|
||||
getClientIp,
|
||||
safeJson,
|
||||
sanitiseZodIssues,
|
||||
} from "@/lib/api-response";
|
||||
import { auditLog } from "@/lib/auth/audit";
|
||||
import { decrypt, encrypt } from "@/lib/crypto";
|
||||
import { prisma } from "@/lib/db";
|
||||
import { sourceClient } from "@/lib/documents/sources/clients";
|
||||
import {
|
||||
loadConnection,
|
||||
toConnectionDto,
|
||||
} from "@/lib/documents/sources/connections";
|
||||
import { DocumentSourceError } from "@/lib/documents/sources/http";
|
||||
import { evaluateSourceBaseUrl } from "@/lib/documents/sources/origins";
|
||||
import {
|
||||
admitDocumentSourceCaller,
|
||||
checkConnectRateLimit,
|
||||
responseForSourceFailure,
|
||||
sourceErrorResponse,
|
||||
systemParam,
|
||||
} from "@/lib/documents/sources/route-support";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import { rateLimitHeaders } from "@/lib/rate-limit";
|
||||
import { documentSourceSaveSchema } from "@/lib/validations/document-sources";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
type RouteParams = { params: Promise<{ system: string }> };
|
||||
|
||||
export const PUT = apiHandler(
|
||||
async (request: NextRequest, { params }: RouteParams) => {
|
||||
const auth = await requireAuth();
|
||||
const refused = await admitDocumentSourceCaller(auth);
|
||||
if (refused) return refused;
|
||||
const system = await systemParam(params);
|
||||
if (system instanceof Response) return system;
|
||||
const userId = auth.user.id;
|
||||
annotate({ action: { name: "documents.sources.save" }, meta: { system } });
|
||||
|
||||
const rl = await checkConnectRateLimit(userId);
|
||||
if (!rl.allowed) {
|
||||
return sourceErrorResponse(
|
||||
"rateLimited",
|
||||
undefined,
|
||||
rateLimitHeaders(rl),
|
||||
);
|
||||
}
|
||||
|
||||
const { data: body, error: jsonError } = await safeJson(request, {
|
||||
maxBytes: 8 * 1024,
|
||||
});
|
||||
if (jsonError) return jsonError;
|
||||
const parsed = documentSourceSaveSchema.safeParse(body);
|
||||
if (!parsed.success) {
|
||||
return apiValidationError(
|
||||
"Invalid connection details",
|
||||
sanitiseZodIssues(parsed.error.issues),
|
||||
422,
|
||||
);
|
||||
}
|
||||
|
||||
const verdict = evaluateSourceBaseUrl(parsed.data.baseUrl);
|
||||
if (!verdict.ok) {
|
||||
return verdict.reason === "notAllowed"
|
||||
? sourceErrorResponse("originNotAllowed")
|
||||
: apiError("Enter the full address, starting with https://.", 422, {
|
||||
errorCode: "documents.sources.invalidAddress",
|
||||
});
|
||||
}
|
||||
|
||||
const organizationId =
|
||||
system === "PAPRA" ? (parsed.data.organizationId ?? null) : null;
|
||||
if (system === "PAPRA" && !organizationId) {
|
||||
return apiError("Papra needs the organization id.", 422, {
|
||||
errorCode: "documents.sources.organizationRequired",
|
||||
});
|
||||
}
|
||||
|
||||
const existing = await loadConnection(userId, system);
|
||||
const token =
|
||||
parsed.data.token ?? (existing ? decrypt(existing.tokenEncrypted) : null);
|
||||
if (!token) {
|
||||
return apiError("Enter the API token.", 422, {
|
||||
errorCode: "documents.sources.tokenRequired",
|
||||
});
|
||||
}
|
||||
|
||||
// Live check before anything is stored: the row only ever holds a
|
||||
// connection that answered once.
|
||||
try {
|
||||
await sourceClient({
|
||||
system,
|
||||
origin: verdict.origin,
|
||||
baseUrl: verdict.baseUrl,
|
||||
organizationId,
|
||||
token,
|
||||
}).test();
|
||||
} catch (err) {
|
||||
if (err instanceof DocumentSourceError) {
|
||||
annotate({ meta: { refused: err.code } });
|
||||
}
|
||||
return responseForSourceFailure(err);
|
||||
}
|
||||
|
||||
const now = new Date();
|
||||
const tokenEncrypted = encrypt(token);
|
||||
// Field by field; `userId` from the session, never the body.
|
||||
const saved = await prisma.documentSourceConnection.upsert({
|
||||
where: { userId_system: { userId, system } },
|
||||
create: {
|
||||
userId,
|
||||
system,
|
||||
baseUrl: verdict.baseUrl,
|
||||
organizationId,
|
||||
tokenEncrypted,
|
||||
lastVerifiedAt: now,
|
||||
},
|
||||
update: {
|
||||
baseUrl: verdict.baseUrl,
|
||||
organizationId,
|
||||
tokenEncrypted,
|
||||
lastVerifiedAt: now,
|
||||
},
|
||||
select: {
|
||||
system: true,
|
||||
baseUrl: true,
|
||||
organizationId: true,
|
||||
lastVerifiedAt: true,
|
||||
},
|
||||
});
|
||||
|
||||
await auditLog("documents.sources.connect", {
|
||||
userId,
|
||||
ipAddress: getClientIp(request),
|
||||
details: { system, origin: verdict.origin, replaced: existing !== null },
|
||||
});
|
||||
|
||||
return apiSuccess(toConnectionDto(saved));
|
||||
},
|
||||
);
|
||||
|
||||
export const DELETE = apiHandler(
|
||||
async (request: NextRequest, { params }: RouteParams) => {
|
||||
const auth = await requireAuth();
|
||||
// No list needed to forget a connection: an operator who removed the
|
||||
// origin must not leave the person unable to delete the stored token.
|
||||
const refused = await admitDocumentSourceCaller(auth, { needsList: false });
|
||||
if (refused) return refused;
|
||||
const system = await systemParam(params);
|
||||
if (system instanceof Response) return system;
|
||||
annotate({
|
||||
action: { name: "documents.sources.disconnect" },
|
||||
meta: { system },
|
||||
});
|
||||
|
||||
const { count } = await prisma.documentSourceConnection.deleteMany({
|
||||
where: { userId: auth.user.id, system },
|
||||
});
|
||||
if (count > 0) {
|
||||
await auditLog("documents.sources.disconnect", {
|
||||
userId: auth.user.id,
|
||||
ipAddress: getClientIp(request),
|
||||
details: { system },
|
||||
});
|
||||
}
|
||||
return apiSuccess({ disconnected: count > 0 });
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,131 @@
|
||||
/**
|
||||
* Document picker (#1038): search one connected document archive.
|
||||
*
|
||||
* `q` searches the document's name (the title in Paperless-ngx, the name in
|
||||
* Papra), `tag` narrows to one of the source's tags by its id, `from` / `to`
|
||||
* to a date range (inclusive), `page` pages through 25 at a time. The answer
|
||||
* is a minimal row per document (source id, title, date, tag names, size when
|
||||
* the source says it) plus what HealthLog already holds under that source key:
|
||||
* `new`, `imported` (with the vault document's id) or `deleted` (the person
|
||||
* deleted it here; importing it again is refused). Nothing is stored.
|
||||
*
|
||||
* Sixty searches a minute, shared with the tag read. Cookie-only and
|
||||
* owner-only; see `admitDocumentSourceCaller`.
|
||||
*/
|
||||
import { NextRequest } from "next/server";
|
||||
|
||||
import { apiHandler, requireAuth } from "@/lib/api-handler";
|
||||
import {
|
||||
apiSuccess,
|
||||
apiValidationError,
|
||||
sanitiseZodIssues,
|
||||
} from "@/lib/api-response";
|
||||
import { findSourceKeyStates } from "@/lib/documents/source-key";
|
||||
import type { SourceListItem } from "@/lib/documents/sources/clients";
|
||||
import { openConnection } from "@/lib/documents/sources/connections";
|
||||
import {
|
||||
admitDocumentSourceCaller,
|
||||
checkSearchRateLimit,
|
||||
responseForSourceFailure,
|
||||
sourceErrorResponse,
|
||||
systemParam,
|
||||
} from "@/lib/documents/sources/route-support";
|
||||
import type {
|
||||
DocumentSourceResultDto,
|
||||
DocumentSourceSearchDto,
|
||||
} from "@/lib/documents/sources/types";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import { rateLimitHeaders } from "@/lib/rate-limit";
|
||||
import { documentSourceSearchSchema } from "@/lib/validations/document-sources";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
type RouteParams = { params: Promise<{ system: string }> };
|
||||
|
||||
export const GET = apiHandler(
|
||||
async (request: NextRequest, { params }: RouteParams) => {
|
||||
const auth = await requireAuth();
|
||||
const refused = await admitDocumentSourceCaller(auth);
|
||||
if (refused) return refused;
|
||||
const system = await systemParam(params);
|
||||
if (system instanceof Response) return system;
|
||||
const userId = auth.user.id;
|
||||
|
||||
const url = new URL(request.url);
|
||||
const parsed = documentSourceSearchSchema.safeParse({
|
||||
q: url.searchParams.get("q") ?? undefined,
|
||||
tag: url.searchParams.get("tag") ?? undefined,
|
||||
from: url.searchParams.get("from") ?? undefined,
|
||||
to: url.searchParams.get("to") ?? undefined,
|
||||
page: url.searchParams.get("page") ?? undefined,
|
||||
});
|
||||
if (!parsed.success) {
|
||||
return apiValidationError(
|
||||
"Invalid search",
|
||||
sanitiseZodIssues(parsed.error.issues),
|
||||
422,
|
||||
);
|
||||
}
|
||||
const { q, tag, from, to, page } = parsed.data;
|
||||
|
||||
const rl = await checkSearchRateLimit(userId);
|
||||
if (!rl.allowed) {
|
||||
return sourceErrorResponse(
|
||||
"rateLimited",
|
||||
undefined,
|
||||
rateLimitHeaders(rl),
|
||||
);
|
||||
}
|
||||
|
||||
let found: { items: SourceListItem[]; hasMore: boolean };
|
||||
try {
|
||||
const client = await openConnection(userId, system);
|
||||
found = await client.search({
|
||||
q,
|
||||
tagId: tag ?? null,
|
||||
from: from ?? null,
|
||||
to: to ?? null,
|
||||
page,
|
||||
});
|
||||
} catch (err) {
|
||||
return responseForSourceFailure(err);
|
||||
}
|
||||
|
||||
const states = await findSourceKeyStates(
|
||||
userId,
|
||||
system,
|
||||
found.items.map((item) => item.sourceId),
|
||||
);
|
||||
const results: DocumentSourceResultDto[] = found.items.map((item) => {
|
||||
const held = states.get(item.sourceId);
|
||||
return {
|
||||
sourceId: item.sourceId,
|
||||
title: item.title,
|
||||
date: item.date,
|
||||
tags: item.tags,
|
||||
sizeBytes: item.sizeBytes,
|
||||
state: held?.state ?? "new",
|
||||
documentId: held?.state === "imported" ? held.documentId : null,
|
||||
};
|
||||
});
|
||||
|
||||
// Counts only: the query and the titles are the person's own words and
|
||||
// stay out of the wide event.
|
||||
annotate({
|
||||
action: { name: "documents.sources.search" },
|
||||
meta: {
|
||||
system,
|
||||
page,
|
||||
count: results.length,
|
||||
filtered: Boolean(q || tag || from || to),
|
||||
},
|
||||
});
|
||||
|
||||
const body: DocumentSourceSearchDto = {
|
||||
results,
|
||||
page,
|
||||
hasMore: found.hasMore,
|
||||
};
|
||||
return apiSuccess(body);
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,52 @@
|
||||
/**
|
||||
* Document picker (#1038): the tags of one connected document archive, for the
|
||||
* picker's tag filter. The first 250 by name, as `{ id, name }` in the
|
||||
* source's own ids. Shares the search allowance (60 a minute).
|
||||
*
|
||||
* Cookie-only and owner-only; see `admitDocumentSourceCaller`.
|
||||
*/
|
||||
import { NextRequest } from "next/server";
|
||||
|
||||
import { apiHandler, requireAuth } from "@/lib/api-handler";
|
||||
import { apiSuccess } from "@/lib/api-response";
|
||||
import { openConnection } from "@/lib/documents/sources/connections";
|
||||
import {
|
||||
admitDocumentSourceCaller,
|
||||
checkSearchRateLimit,
|
||||
responseForSourceFailure,
|
||||
sourceErrorResponse,
|
||||
systemParam,
|
||||
} from "@/lib/documents/sources/route-support";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import { rateLimitHeaders } from "@/lib/rate-limit";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
type RouteParams = { params: Promise<{ system: string }> };
|
||||
|
||||
export const GET = apiHandler(
|
||||
async (_request: NextRequest, { params }: RouteParams) => {
|
||||
const auth = await requireAuth();
|
||||
const refused = await admitDocumentSourceCaller(auth);
|
||||
if (refused) return refused;
|
||||
const system = await systemParam(params);
|
||||
if (system instanceof Response) return system;
|
||||
annotate({ action: { name: "documents.sources.tags" }, meta: { system } });
|
||||
|
||||
const rl = await checkSearchRateLimit(auth.user.id);
|
||||
if (!rl.allowed) {
|
||||
return sourceErrorResponse(
|
||||
"rateLimited",
|
||||
undefined,
|
||||
rateLimitHeaders(rl),
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const client = await openConnection(auth.user.id, system);
|
||||
return apiSuccess({ tags: await client.tags() });
|
||||
} catch (err) {
|
||||
return responseForSourceFailure(err);
|
||||
}
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,71 @@
|
||||
/**
|
||||
* Document picker (#1038): test the saved connection to one document archive.
|
||||
*
|
||||
* One request to the source with the stored token, answered `{ ok, latencyMs }`
|
||||
* or with the reason it failed (`meta.errorCode`, and `meta.upstreamStatus`
|
||||
* when the source answered at all; its body is never passed on). A success
|
||||
* stamps `lastVerifiedAt`. Shares the save's allowance of ten a minute.
|
||||
*
|
||||
* Cookie-only and owner-only; see `admitDocumentSourceCaller`.
|
||||
*/
|
||||
import { NextRequest } from "next/server";
|
||||
|
||||
import { apiHandler, requireAuth } from "@/lib/api-handler";
|
||||
import { apiSuccess } from "@/lib/api-response";
|
||||
import {
|
||||
markVerified,
|
||||
openConnection,
|
||||
} from "@/lib/documents/sources/connections";
|
||||
import { DocumentSourceError } from "@/lib/documents/sources/http";
|
||||
import {
|
||||
admitDocumentSourceCaller,
|
||||
checkConnectRateLimit,
|
||||
responseForSourceFailure,
|
||||
sourceErrorResponse,
|
||||
systemParam,
|
||||
} from "@/lib/documents/sources/route-support";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import { rateLimitHeaders } from "@/lib/rate-limit";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
type RouteParams = { params: Promise<{ system: string }> };
|
||||
|
||||
export const POST = apiHandler(
|
||||
async (_request: NextRequest, { params }: RouteParams) => {
|
||||
const auth = await requireAuth();
|
||||
const refused = await admitDocumentSourceCaller(auth);
|
||||
if (refused) return refused;
|
||||
const system = await systemParam(params);
|
||||
if (system instanceof Response) return system;
|
||||
const userId = auth.user.id;
|
||||
annotate({ action: { name: "documents.sources.test" }, meta: { system } });
|
||||
|
||||
const rl = await checkConnectRateLimit(userId);
|
||||
if (!rl.allowed) {
|
||||
return sourceErrorResponse(
|
||||
"rateLimited",
|
||||
undefined,
|
||||
rateLimitHeaders(rl),
|
||||
);
|
||||
}
|
||||
|
||||
const start = performance.now();
|
||||
try {
|
||||
const client = await openConnection(userId, system);
|
||||
await client.test();
|
||||
} catch (err) {
|
||||
if (err instanceof DocumentSourceError) {
|
||||
annotate({ meta: { refused: err.code } });
|
||||
}
|
||||
return responseForSourceFailure(err);
|
||||
}
|
||||
const latencyMs = Math.round(performance.now() - start);
|
||||
const verifiedAt = await markVerified(userId, system);
|
||||
return apiSuccess({
|
||||
ok: true,
|
||||
latencyMs,
|
||||
lastVerifiedAt: verifiedAt.toISOString(),
|
||||
});
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,54 @@
|
||||
/**
|
||||
* Document picker (#1038): which document archives this person can pick from.
|
||||
*
|
||||
* `available` says whether the operator listed any origin in
|
||||
* `DOCUMENT_SOURCE_ORIGINS`; `allowedOrigins` lists them so the settings card
|
||||
* can say which addresses will be accepted; `connections` are the person's own
|
||||
* saved connections, never with their token (`hasToken` is the literal true).
|
||||
* No request leaves the server here.
|
||||
*
|
||||
* Cookie-only and owner-only like every route under this path
|
||||
* (`admitDocumentSourceCaller`). With the list unset it answers 200
|
||||
* `available: false` rather than 404, because this is the one read the client
|
||||
* makes to decide whether to show the picker at all.
|
||||
*/
|
||||
import { apiHandler, requireAuth } from "@/lib/api-handler";
|
||||
import { apiSuccess } from "@/lib/api-response";
|
||||
import {
|
||||
listConnections,
|
||||
toConnectionDto,
|
||||
} from "@/lib/documents/sources/connections";
|
||||
import {
|
||||
documentSourcesAvailable,
|
||||
listedDocumentSourceOrigins,
|
||||
} from "@/lib/documents/sources/origins";
|
||||
import type { DocumentSourcesStatusDto } from "@/lib/documents/sources/types";
|
||||
import { admitDocumentSourceCaller } from "@/lib/documents/sources/route-support";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
export const GET = apiHandler(async () => {
|
||||
const auth = await requireAuth();
|
||||
const refused = await admitDocumentSourceCaller(auth, { needsList: false });
|
||||
if (refused) return refused;
|
||||
|
||||
annotate({ action: { name: "documents.sources.status" } });
|
||||
|
||||
if (!documentSourcesAvailable()) {
|
||||
const off: DocumentSourcesStatusDto = {
|
||||
available: false,
|
||||
allowedOrigins: [],
|
||||
connections: [],
|
||||
};
|
||||
return apiSuccess(off);
|
||||
}
|
||||
|
||||
const rows = await listConnections(auth.user.id);
|
||||
const status: DocumentSourcesStatusDto = {
|
||||
available: true,
|
||||
allowedOrigins: listedDocumentSourceOrigins(),
|
||||
connections: rows.map(toConnectionDto),
|
||||
};
|
||||
return apiSuccess(status);
|
||||
});
|
||||
Reference in new issue
Block a user