App 表面数据采集设计笔记

function resolveProductInput(input):
  text = trim(input)

  if text contains direct goods id:
    return { itemId, source: "direct" }

  urls = extractUrls(text)

  for url in urls:
    if url contains goods id:
      return { itemId, source: "url", resolvedUrl: url }

    if url is short link:
      finalUrl = followRedirect(url)
      if finalUrl contains goods id:
        return { itemId, source: "short_link", resolvedUrl: finalUrl }

  raise "no product id found"

function addProduct(input, includeDetail):
  resolved = resolveProductInput(input)
  product = upsertProduct(resolved.itemId)

  product.webStatus = "running"
  product = refreshWebSnapshot(product)

  if includeDetail:
    enqueueAppDetailTask(product)
    product.detailStatus = "pending"

  return product

function collectMobileOnlySignals(product):
  device = selectIdleDevice(product.preferredDeviceId)
  if device is null:
    enqueue(product)
    return "pending"

  with adbSession(device.serial):
    openProductInApp(product.itemId)
    xml = dumpUi()
    tags = parseVisibleTags(xml)
    saveEvidence(product, xml, tags, device)

function selectIdleDevice(preferredDeviceId = null):
  statuses = adbDevices()
  busy = queryRunningTaskDeviceIds()
  devices = activeDevicesOrderedById()

  if preferredDeviceId exists:
    device = find(devices, preferredDeviceId)
    if device is online and device.id not in busy:
      return device
    return null

  for device in devices:
    if device.id in busy:
      continue
    if statuses[device.serial] == "device":
      return device

  return null

function dispatchPendingTasks():
  tasks = oldestPendingTasks(limit = 10)

  for task in tasks:
    device = selectIdleDevice(task.preferredDeviceId)
    if device is null:
      continue

    task.status = "running"
    task.deviceId = device.id
    task.startedAt = now()
    save(task)

    runTask(task.id)

function runTask(taskId):
  task = loadTask(taskId)
  device = loadDevice(task.deviceId)

  try:
    result = collectFromApp(task.itemId, device.serial)
    task.status = "success"
    task.tags = result.tags
    task.rawItems = result.items
    task.elapsedMs = result.elapsedMs
    task.finishedAt = now()
    markProductDetailSuccess(task.productId)
    saveMetrics(task.productId, result.tags)
    markDeviceSeen(device)
  catch error:
    task.status = "failed"
    task.error = error.message
    task.finishedAt = now()
    markProductDetailFailed(task.productId)

  save(task)
  dispatchPendingTasks()

function collectFromApp(itemId, serial):
  openDeepLink(serial, "app://goods_detail/" + itemId)
  waitForSettledScreen()

  screen = getScreenSize(serial)
  allItems = []
  seen = set()
  lastTagY = null

  for round in 0..maxSwipes:
    xml = dumpUiXml(serial)

    if pageShowsLoadError(xml):
      raise "product page load failed"

    items = extractCandidateTags(xml, screen.height)

    for item in items:
      allItems.append(item)
      seen.add(item.text)

    if noNewTagsFound(seen):
      break

    lastTagY = medianY(items)
    swipeTagArea(serial, y = lastTagY)
    waitForSettle()

  return uniqueByText(allItems)

patterns = [
  ("24h_cart", /24小时内(\d+)\+?人加购/),
  ("7d_fav", /近7天新增(\d+)\+?人收藏/)
]

function saveMetrics(product, tags):
  for tag in tags:
    for dimension, pattern in patterns:
      if pattern matches tag:
        insertMetric(product.id, product.itemId, dimension, capturedNumber)

function createKey(name, expiresAt, permissions):
  raw = "sk-" + secureRandom()

  record = {
    name,
    keyPrefix: firstVisiblePart(raw),
    keyHash: sha256(raw),
    expiresAt,
    permissions,
    status: "active"
  }

  save(record)
  return raw

function verifyKey(raw):
  if raw does not start with "sk-":
    return null

  record = findByHash(sha256(raw))

  if record is missing or revoked or expired:
    return null

  record.lastUsedAt = now()
  return record

normalize early, collect in layers, store evidence, extract metrics late