Synthetic datasets

DATASET_ID="synthetic-finnish-vehicle-market"
RELEASE_ID="replace-with-accepted-release-id"

# Inspect the accepted release and its metadata first.
curl --fail --location \
  "https://api.datasets.nordicdevhouse.com/v1/datasets/$DATASET_ID/releases/$RELEASE_ID" \
  -H "Authorization: Bearer $DATASET_API_KEY" \
  -H "X-API-Version: 1"

# Download only after the release status and dataset label are accepted.
curl --fail --location \
  "https://api.datasets.nordicdevhouse.com/v1/datasets/$DATASET_ID/releases/$RELEASE_ID/download?format=parquet" \
  -H "Authorization: Bearer $DATASET_API_KEY" \
  -H "X-API-Version: 1" \
  --output "$RELEASE_ID.parquet"

import os
from pathlib import Path
import requests

base_url = "https://api.datasets.nordicdevhouse.com/v1"
dataset_id = os.environ["DATASET_ID"]
release_id = os.environ["RELEASE_ID"]
headers = {
    "Authorization": f"Bearer {os.environ['DATASET_API_KEY']}",
    "X-API-Version": "1",
}

release = requests.get(
    f"{base_url}/datasets/{dataset_id}/releases/{release_id}",
    headers=headers,
    timeout=20,
)
release.raise_for_status()
manifest = release.json()
if manifest.get("quality_status") != "complete":
    raise RuntimeError("Release is not accepted for delivery")

target = Path(f"{release_id}.jsonl.part")
with requests.get(
    f"{base_url}/datasets/{dataset_id}/releases/{release_id}/download?format=jsonl",
    headers=headers,
    timeout=120,
    stream=True,
) as response:
    response.raise_for_status()
    with target.open("wb") as output:
        for chunk in response.iter_content(chunk_size=1024 * 1024):
            if chunk:
                output.write(chunk)

target.replace(target.with_suffix(""))

import { createWriteStream } from "node:fs";
import { pipeline } from "node:stream/promises";
import { Readable } from "node:stream";

const baseUrl = "https://api.datasets.nordicdevhouse.com/v1";
const { DATASET_API_KEY, DATASET_ID, RELEASE_ID } = process.env;
const headers = {
  Authorization: "Bearer " + DATASET_API_KEY,
  "X-API-Version": "1",
};

const releaseUrl = baseUrl + "/datasets/" + DATASET_ID + "/releases/" + RELEASE_ID;
const releaseResponse = await fetch(releaseUrl, { headers });
if (!releaseResponse.ok) throw new Error("Release lookup failed: " + releaseResponse.status);
const release = await releaseResponse.json();
if (release.quality_status !== "complete") throw new Error("Release is not accepted for delivery");

const download = await fetch(releaseUrl + "/download?format=jsonl", { headers });
if (!download.ok || !download.body) throw new Error("Download failed: " + download.status);
await pipeline(Readable.fromWeb(download.body), createWriteStream(RELEASE_ID + ".jsonl"));