fix(observability,webhooks,i18n): timestamped logs, dedupe webhook retries, default non-de locales to English

- New custom server.js (replaces react-router-serve): ISO timestamps on
  all console.* output and on access logs, and skip successful /healthz
  polls so real traffic stays visible.
- New ProcessedWebhook table + dedupe helper keyed on
  X-Shopify-Webhook-Id; stops Shopify retries from triggering a second
  invoice email when the original delivery exceeded the 5s ack timeout.
- orders/create + orders/fulfilled now respond 200 immediately and run
  the PDF/email work in the background so we stay under that timeout.
- pickLanguage(): non-German locales (it, fr, es, ...) now default to
  English instead of falling back to German. Empty/unknown still maps to
  'de' so the per-shop defaultLanguage chain keeps working.
- Tests for pickLanguage and dedupe via node --test + tsx.

app/routes/webhooks.orders.create.tsx

@@ -5,6 +5,8 @@ import {
   generateAndEmailInvoice,
   isManualPaymentOrder,
 } from "../services/invoice/automations.server";
+import { isDuplicateWebhook } from "../services/webhooks/dedupe.server";
+import { runWebhookInBackground } from "../services/webhooks/background.server";
 
 /**
  * orders/create — Automation 1: when a wire-transfer (manual-payment-gateway)
@@ -16,31 +18,42 @@ export const action = async ({ request }: ActionFunctionArgs) => {
   const { shop, topic, payload, session, admin } = await authenticate.webhook(request);
   console.log(`Received ${topic} webhook for ${shop}`);
 
-  if (!session || !admin) return new Response();
+  // Drop Shopify retries we've already processed (prevents the duplicate
+  // invoice email we saw when the first delivery exceeded Shopify's 5s ack
+  // timeout — the work still completed, but Shopify resent the webhook).
+  if (await isDuplicateWebhook(request, shop, topic)) return new Response();
 
-  const settings = await db.shopSettings.findUnique({ where: { shopDomain: shop } });
-  if (!settings?.autoEmailOnWireTransferPlaced) return new Response();
+  if (!session || !admin) return new Response();
 
   const orderId = payload?.id;
   if (orderId == null) return new Response();
 
-  const orderGid = `gid://shopify/Order/${String(orderId).replace(/^.*\//, "")}`;
-  const isManual = await isManualPaymentOrder(admin, orderGid);
-  if (!isManual) return new Response();
+  const customerLocale =
+    typeof payload?.customer_locale === "string" ? payload.customer_locale : undefined;
+
+  // Respond 200 immediately and run the (slow) PDF + email work in the
+  // background — keeps us well under Shopify's ~5s ack timeout. Dedupe
+  // above guarantees we don't double-process a retry while the first
+  // delivery is still working.
+  runWebhookInBackground(`${topic} order=${orderId} shop=${shop}`, async () => {
+    const settings = await db.shopSettings.findUnique({ where: { shopDomain: shop } });
+    if (!settings?.autoEmailOnWireTransferPlaced) return;
+
+    const orderGid = `gid://shopify/Order/${String(orderId).replace(/^.*\//, "")}`;
+    if (!(await isManualPaymentOrder(admin, orderGid))) return;
 
-  try {
     const result = await generateAndEmailInvoice({
       shopDomain: shop,
       admin,
       orderId,
-      customerLocale: typeof payload?.customer_locale === "string" ? payload.customer_locale : undefined,
+      customerLocale,
     });
     if (!result.ok) {
-      console.warn(`auto-email (wire-transfer placed) failed for order ${orderId} on ${shop}: ${result.reason}`);
+      console.warn(
+        `auto-email (wire-transfer placed) failed for order ${orderId} on ${shop}: ${result.reason}`,
+      );
     }
-  } catch (err) {
-    console.error(`auto-email (wire-transfer placed) crashed for order ${orderId} on ${shop}:`, err);
-  }
+  });
 
   return new Response();
 };

app/routes/webhooks.orders.fulfilled.tsx

@@ -5,6 +5,8 @@ import {
   generateAndEmailInvoice,
   isManualPaymentOrder,
 } from "../services/invoice/automations.server";
+import { isDuplicateWebhook } from "../services/webhooks/dedupe.server";
+import { runWebhookInBackground } from "../services/webhooks/background.server";
 
 /**
  * orders/fulfilled — Automation 2: when an order is fulfilled and is NOT a
@@ -16,36 +18,42 @@ export const action = async ({ request }: ActionFunctionArgs) => {
   const { shop, topic, payload, session, admin } = await authenticate.webhook(request);
   console.log(`Received ${topic} webhook for ${shop}`);
 
+  // Idempotency against Shopify retries — see webhooks/dedupe.server.ts.
+  if (await isDuplicateWebhook(request, shop, topic)) return new Response();
+
   if (!session || !admin) {
     // App was uninstalled before the webhook drained — nothing to do.
     return new Response();
   }
 
-  const settings = await db.shopSettings.findUnique({ where: { shopDomain: shop } });
-  if (!settings?.autoEmailOnFulfilledNonWireTransfer) return new Response();
-
   const orderId = payload?.id;
   if (orderId == null) return new Response();
 
-  const orderGid = `gid://shopify/Order/${String(orderId).replace(/^.*\//, "")}`;
-  if (await isManualPaymentOrder(admin, orderGid)) {
-    // Manual / wire-transfer order — handled by Automation 1, skip here.
-    return new Response();
-  }
+  const customerLocale =
+    typeof payload?.customer_locale === "string" ? payload.customer_locale : undefined;
+
+  // Respond fast; do the heavy lifting after the response (see notes in
+  // webhooks.orders.create.tsx for the rationale).
+  runWebhookInBackground(`${topic} order=${orderId} shop=${shop}`, async () => {
+    const settings = await db.shopSettings.findUnique({ where: { shopDomain: shop } });
+    if (!settings?.autoEmailOnFulfilledNonWireTransfer) return;
+
+    const orderGid = `gid://shopify/Order/${String(orderId).replace(/^.*\//, "")}`;
+    if (await isManualPaymentOrder(admin, orderGid)) {
+      // Manual / wire-transfer order — handled by Automation 1, skip here.
+      return;
+    }
 
-  try {
     const result = await generateAndEmailInvoice({
       shopDomain: shop,
       admin,
       orderId,
-      customerLocale: typeof payload?.customer_locale === "string" ? payload.customer_locale : undefined,
+      customerLocale,
     });
     if (!result.ok) {
       console.warn(`auto-email (fulfilled) failed for order ${orderId} on ${shop}: ${result.reason}`);
     }
-  } catch (err) {
-    console.error(`auto-email (fulfilled) crashed for order ${orderId} on ${shop}:`, err);
-  }
+  });
 
   return new Response();
 };

app/routes/webhooks.orders.updated.tsx

@@ -1,10 +1,13 @@
 import type { ActionFunctionArgs } from "react-router";
 import { authenticate } from "../shopify.server";
+import { isDuplicateWebhook } from "../services/webhooks/dedupe.server";
 
 // Acknowledged but not yet acted on. Future: invalidate cached invoice
 // snapshots when a relevant field on the order changes.
 export const action = async ({ request }: ActionFunctionArgs) => {
   const { shop, topic } = await authenticate.webhook(request);
   console.log(`Received ${topic} webhook for ${shop}`);
+  // Idempotency against Shopify retries — see webhooks/dedupe.server.ts.
+  if (await isDuplicateWebhook(request, shop, topic)) return new Response();
   return new Response();
 };

app/services/invoice/i18n.ts

@@ -167,11 +167,19 @@ const en: InvoiceStrings = {
   paidStamp: "PAID",
 };
 
+// Locale → invoice language. We only render in German (`de`) when the
+// caller is explicitly German-speaking (de, de-AT, de-DE, de_CH, …).
+// Everything else (it, fr, es, en, …) falls back to English so that
+// non-German-speaking customers don't receive a German invoice. Callers
+// that have a per-shop default fall back to it via
+// `pickLanguage(customerLocale ?? settings.defaultLanguage)`, which is why
+// `null`/`undefined` still maps to `de` (the legacy default for the
+// Austrian shops this app was built for).
 export function pickLanguage(input: string | null | undefined): InvoiceLanguage {
   if (!input) return "de";
   const v = input.toLowerCase();
-  if (v.startsWith("en")) return "en";
-  return "de";
+  if (v.startsWith("de")) return "de";
+  return "en";
 }
 
 export function getStrings(language: InvoiceLanguage): InvoiceStrings {

app/services/webhooks/background.server.ts

@@ -0,0 +1,27 @@
+/**
+ * Fire-and-forget runner for webhook side-effects.
+ *
+ * Shopify expects a 200 response within ~5 seconds, otherwise it considers
+ * the delivery failed and retries it. Heavy automation work (PDF render,
+ * Shopify Files upload, SMTP send) routinely exceeded that budget, which
+ * caused duplicate invoice emails before we added the dedupe table.
+ *
+ * Returning the response immediately and letting the work finish in the
+ * background keeps Shopify happy. Combined with the dedupe table this is
+ * defence-in-depth: dedupe ensures *correctness* even if a retry sneaks
+ * through, while async processing makes retries unlikely in the first
+ * place.
+ *
+ * Errors are caught and logged \u2014 they cannot reach a dispatcher because
+ * the HTTP response is already gone.
+ */
+export function runWebhookInBackground(
+  description: string,
+  work: () => Promise<unknown>,
+): void {
+  // `void` so we don't accidentally `await` the floating promise; the
+  // node event loop keeps the task alive until it settles.
+  void work().catch((err) => {
+    console.error(`background webhook task '${description}' failed:`, err);
+  });
+}

app/services/webhooks/dedupe.server.ts

@@ -0,0 +1,67 @@
+import db from "../../db.server";
+
+/**
+ * Minimal shape of the Prisma client surface we use — declared inline so
+ * the helper can be unit-tested with a tiny stub instead of pulling in a
+ * real database.
+ */
+export interface DedupeDeps {
+  db: {
+    processedWebhook: {
+      create: (args: {
+        data: { webhookId: string; topic: string; shopDomain: string };
+      }) => Promise<unknown>;
+    };
+  };
+}
+
+/**
+ * Returns `true` when this Shopify webhook delivery has already been
+ * processed and the caller should short-circuit without doing the work.
+ *
+ * Shopify retries webhook deliveries when it doesn't receive a 200 within
+ * its (~5s) timeout window. Without dedupe this caused us to email an
+ * invoice twice for the same order: the first slow delivery completed its
+ * work but Shopify timed out and re-sent the webhook, which then ran the
+ * automation a second time.
+ *
+ * We key on the `X-Shopify-Webhook-Id` header — Shopify guarantees the same
+ * value for retries of the same delivery, but a new value for genuinely
+ * new events. The insert is the lock: a unique-constraint violation
+ * (Prisma error code `P2002`) means another delivery already claimed this
+ * id.
+ */
+export async function isDuplicateWebhook(
+  request: Request,
+  shop: string,
+  topic: string,
+  deps: DedupeDeps = { db },
+): Promise<boolean> {
+  const webhookId = request.headers.get("x-shopify-webhook-id");
+  if (!webhookId) {
+    // Defensive: in unit tests / non-Shopify callers there is no id.
+    // Don't dedupe — that would silently drop legitimate calls.
+    return false;
+  }
+  try {
+    await deps.db.processedWebhook.create({
+      data: { webhookId, topic, shopDomain: shop },
+    });
+    return false;
+  } catch (err) {
+    // Duck-typed P2002 check so callers can stub the db without pulling
+    // in the real `Prisma` namespace.
+    if ((err as { code?: string } | null)?.code === "P2002") {
+      console.log(
+        `dedupe: skipping duplicate ${topic} delivery for ${shop} (webhookId=${webhookId})`,
+      );
+      return true;
+    }
+    // Don't fail the webhook on a logging-table issue; just process it.
+    console.warn(
+      `dedupe: failed to record webhook ${webhookId} (${topic}/${shop}):`,
+      err,
+    );
+    return false;
+  }
+}