Skip to content

Store

Workspace

Workspace(
    database: str = ":memory:",
    *,
    con: DuckDBPyConnection | None = None,
    name: str = "datarecord",
)

A DuckDB database and the files it reaches, shared by every store on it.

con is the calling thread's cursor: one workspace serves every thread, and their work shares one database - its tables, never its relations.

Parameters:

Name Type Description Default
database str

DuckDB database, :memory: by default. What lives there is a workspace and never the record: deleting it loses at most uncommitted edits.

':memory:'
con DuckDBPyConnection | None

A connection to attach database on, such as a server's; its default database and transactions stay its own. A fresh in-memory one by default.

None
name str

What database is attached as, and named by in SQL.

'datarecord'

Raises:

Type Description
BinderException

If name is a database already attached to the connection.

Notes
Source code in src/datarecord/store/store.py
def __init__(
    self,
    database: str = ":memory:",
    *,
    con: DuckDBPyConnection | None = None,
    name: str = "datarecord",
) -> None:
    self.name = name
    self._owned = con is None
    self._db = duckdb.connect() if con is None else con
    with self._db.cursor() as setup:
        setup.execute(f"ATTACH {lit(database)} AS {col(name)}")
        for ddl in (_SCHEMAS, _REVISIONS, _REGISTERED):
            setup.execute(_in(self, ddl))
        for schema in (FOLD, STAGED):
            setup.execute(f"CREATE SCHEMA IF NOT EXISTS {col(name, schema)}")
        registered = setup.execute(
            _in(self, "SELECT name, held FROM {registered}")
        ).fetchall()
    self._made: WeakSet[DuckDBPyConnection] = WeakSet()
    self._made_lock = threading.Lock()
    self._cursors = _Cursors(self)
    self._holds = _Holds(self._db, name)
    for held_by, held in registered:
        self._holds.hold(held_by, held)
    self._stores: dict[str, Store] = {}
    self._schemas: dict[str, Schema] = {}
    self._local = LocalFiles()
    self._remote: Files | None = None
    self._remote_lock = threading.Lock()

con property

con: DuckDBPyConnection

The calling thread's cursor, made on its first use in that thread.

qualified

qualified(table: str) -> str

table in the main schema of the workspace's database, named in full.

Notes
Source code in src/datarecord/store/store.py
def qualified(self, table: str) -> str:
    """`table` in the `main` schema of the workspace's database, named in full.

    Notes
    -----
    - [the workspace's database](https://energy-models.github.io/datarecord/reference/store/#the-database)
    """
    return str(col(self.name, "main", table))

unregister

unregister(name: str) -> None

Drop the schema name a record registered, with its views and its registered() row.

A closed WorkingRecord's staging tables go with its last registration.

Raises:

Type Description
CatalogException

If name is not registered, so a schema no record registered is never dropped.

TransactionException

Where the cursor already holds an open transaction.

Notes
Source code in src/datarecord/store/store.py
def unregister(self, name: str) -> None:
    """Drop the schema `name` a record registered, with its views and its `registered()` row.

    A closed `WorkingRecord`'s staging tables go with its last registration.

    Raises
    ------
    duckdb.CatalogException
        If `name` is not registered, so a schema no record registered is
        never dropped.
    duckdb.TransactionException
        Where the cursor already holds an open transaction.

    Notes
    -----
    - [registered views](https://energy-models.github.io/datarecord/reference/format/#registered-views)
    """
    con = self.con
    with transaction(
        con,
        "unregister",
        "https://energy-models.github.io/datarecord/reference/format/#registered-views",
    ):
        deleted = con.execute(
            _in(self, "DELETE FROM {registered} WHERE name = ? RETURNING name"),
            [name],
        ).fetchall()
        if not deleted:
            msg = f"no record is registered as `{name}` (`Workspace.registered()`)"
            raise duckdb.CatalogException(msg)
        con.execute(f"DROP SCHEMA {col(self.name, name)} CASCADE")
    self._holds.release(name)

registered

registered() -> DuckDBPyRelation

Every registered name, with the revision it reads, whether it is working, the schemas it held, and when it was created.

Notes
Source code in src/datarecord/store/store.py
def registered(self) -> DuckDBPyRelation:
    """Every registered name, with the `revision` it reads, whether it is `working`, the schemas it `held`, and when it was `created`.

    Notes
    -----
    - [registered views](https://energy-models.github.io/datarecord/reference/format/#registered-views)
    """
    return self.con.table(_in(self, "{registered}"))

process classmethod

process() -> Workspace

The process's workspace, created on first use on :memory:.

What a deserialised Revision or Claim attaches to, having no workspace passed; nothing else reaches it unless named.

Notes
Source code in src/datarecord/store/store.py
@classmethod
def process(cls) -> Workspace:
    """The process's workspace, created on first use on `:memory:`.

    What a deserialised `Revision` or `Claim` attaches to, having no
    workspace passed; nothing else reaches it unless named.

    Notes
    -----
    - [crossing a process boundary](https://energy-models.github.io/datarecord/reference/store/#revision)
    """
    global _process
    with _process_lock:
        if _process is None:
            _process = cls()
        return _process

files

files(uri: str) -> Files

The Files for uri's scheme: local without ://, remote for s3://.

Raises:

Type Description
ValueError

For any other scheme.

Source code in src/datarecord/store/store.py
def files(self, uri: str) -> Files:
    """The `Files` for `uri`'s scheme: local without `://`, remote for `s3://`.

    Raises
    ------
    ValueError
        For any other scheme.
    """
    if "://" not in uri:
        return self._local
    if not uri.startswith("s3://"):
        msg = f"{uri}: a location is a local path or an s3:// URI"
        raise ValueError(msg)
    with self._remote_lock:
        if self._remote is None:
            self._remote = self._connect_remote()
        return self._remote

store

store(uri: str) -> Store

This workspace's store at uri, one object per normalised location.

Raises:

Type Description
ValueError

For an empty uri.

Source code in src/datarecord/store/store.py
def store(self, uri: str) -> Store:
    """This workspace's store at `uri`, one object per normalised location.

    Raises
    ------
    ValueError
        For an empty `uri`.
    """
    if not uri:
        msg = (
            "a store needs a location; an empty one would be the working directory"
        )
        raise ValueError(msg)
    location = _absolute(uri)
    self.files(location)
    return self._stores.setdefault(location, Store(location, self))

at

at(uri: str) -> Record

The standalone record at uri, read under its own manifest.json.

A local uri is resolved, as store resolves one, so a later chdir cannot point it at another record.

Raises:

Type Description
FileNotFoundError

If uri has no manifest, as where its write failed.

ValueError

If the manifest names a parent: a layer of a tree is read alone through its store's layer(id).

Notes
Source code in src/datarecord/store/store.py
def at(self, uri: str) -> Record:
    """The standalone record at `uri`, read under its own `manifest.json`.

    A local `uri` is resolved, as `store` resolves one, so a later `chdir`
    cannot point it at another record.

    Raises
    ------
    FileNotFoundError
        If `uri` has no manifest, as where its write failed.
    ValueError
        If the manifest names a parent: a layer of a tree is read alone
        through its store's `layer(id)`.

    Notes
    -----
    - [the record format](https://energy-models.github.io/datarecord/reference/format/)
    """
    base = _absolute(uri) + "/"
    manifest = Manifest.read(
        self.files(base), base + "manifest.json", self._schemas
    )
    if manifest is None:
        msg = f"{base} has no manifest.json, so holds no record"
        raise FileNotFoundError(msg)
    if manifest.parent is not None:
        msg = (
            f"{base} is a layer over revision {manifest.parent}; a layer of a "
            f"tree is read alone through its store's `layer(id)`"
        )
        raise ValueError(msg)
    return _one_layer(base, manifest, self, manifest.id)

write

write(
    source: LayerData | RecordLike,
    uri: str,
    *,
    meta: Meta | None = None,
) -> Record

Write source as a standalone record at uri, and read it back.

A standalone record is one layer at its own URI, written to a URI that does not exist yet: its data first, then its manifest, with a fresh id and no parent, so one whose write failed reads as no record and one written again elsewhere is never read as the one before it. A failed write leaves its data files at uri, locally as remotely, for the caller to delete before writing there again.

Raises:

Type Description
FileExistsError

If anything exists at uri: locally the path, even an empty directory; remotely any key under it.

Notes
Source code in src/datarecord/store/store.py
def write(
    self, source: LayerData | RecordLike, uri: str, *, meta: Meta | None = None
) -> Record:
    """Write `source` as a standalone record at `uri`, and read it back.

    A standalone record is one layer at its own URI, written to a URI that
    does not exist yet: its data first, then its manifest, with a fresh id
    and no parent, so one whose write failed reads as no record and one
    written again elsewhere is never read as the one before it. A failed
    write leaves its data files at `uri`, locally as remotely, for the
    caller to delete before writing there again.

    Raises
    ------
    FileExistsError
        If anything exists at `uri`: locally the path, even an empty
        directory; remotely any key under it.

    Notes
    -----
    - [writing a whole record](https://energy-models.github.io/datarecord/reference/layers/)
    - [a standalone directory](https://energy-models.github.io/datarecord/reference/store/#a-standalone-directory)
    """
    base = _absolute(uri) + "/"
    meta = validate_meta(meta)
    if self.files(base).exists(base):
        msg = (
            f"{base} already exists; a standalone record is written to a URI "
            f"that does not exist yet"
        )
        raise FileExistsError(msg)
    data = layer_data(source, self.con)
    schema = data.schema
    files = write_layer(data, base, self)
    manifest = Manifest(
        id=uuid4(), parent=None, schema=schema, files=files, meta=meta
    )
    manifest.write(self.files(base), base + "manifest.json")
    return self.at(base)

close

close() -> None

Close every thread's cursor and detach the database.

The connection is closed with them only where the workspace opened it; one passed in stays open for its owner.

Source code in src/datarecord/store/store.py
def close(self) -> None:
    """Close every thread's cursor and detach the database.

    The connection is closed with them only where the workspace opened it;
    one passed in stays open for its owner.
    """
    self._holds.close()
    with self._made_lock:
        for cursor in list(self._made):
            cursor.close()
    with self._db.cursor() as teardown:
        teardown.execute(f"DETACH {col(self.name)}")
    if self._owned:
        self._db.close()

Store

Store(uri: str, workspace: Workspace)

One store location: its layers, each carrying the manifest that makes it a revision.

Holds no connection of its own: every read and write goes through its workspace, which several stores share.

Attributes:

Name Type Description
uri

The location: absolute, never empty.

Notes
Source code in src/datarecord/store/store.py
def __init__(self, uri: str, workspace: Workspace) -> None:
    self.uri = uri
    self.workspace = workspace

layer_dir

layer_dir(id: UUID | str) -> str

Revision id's layer directory, with a trailing slash.

Notes
Source code in src/datarecord/store/store.py
def layer_dir(self, id: UUID | str) -> str:
    """Revision `id`'s layer directory, with a trailing slash.

    Notes
    -----
    - [the record format](https://energy-models.github.io/datarecord/reference/format/)
    """
    return f"{self.uri}/layers/{id}/"

manifest_uri

manifest_uri(id: UUID | str) -> str

Layer id's manifest, which makes it a revision.

Source code in src/datarecord/store/store.py
def manifest_uri(self, id: UUID | str) -> str:
    """Layer `id`'s manifest, which makes it a revision."""
    return f"{self.layer_dir(id)}manifest.json"

claim_uri

claim_uri(id: UUID | str) -> str

Layer id's claim, which handed its id out once.

Source code in src/datarecord/store/store.py
def claim_uri(self, id: UUID | str) -> str:
    """Layer `id`'s claim, which handed its id out once."""
    return f"{self.layer_dir(id)}claim.json"

axis_uri

axis_uri(id: UUID | str, dim: str) -> str

Where revision id's layer holds dim's axis.

Notes
Source code in src/datarecord/store/store.py
def axis_uri(self, id: UUID | str, dim: str) -> str:
    """Where revision `id`'s layer holds `dim`'s axis.

    Notes
    -----
    - [the record format](https://energy-models.github.io/datarecord/reference/format/)
    """
    return f"{self.layer_dir(id)}dims/{dim}.parquet"

resolved_dir

resolved_dir(id: UUID | str) -> str

Revision id's node caches, under its layer where no manifest lists them.

Notes
Source code in src/datarecord/store/store.py
def resolved_dir(self, id: UUID | str) -> str:
    """Revision `id`'s node caches, under its layer where no manifest lists them.

    Notes
    -----
    - [materialised node caches](https://energy-models.github.io/datarecord/reference/resolution/#materialised-revisions)
    """
    return f"{self.layer_dir(id)}resolved/"

owner_map_uri

owner_map_uri(id: UUID | str) -> str

Where revision id's owner map is materialised.

Notes
Source code in src/datarecord/store/store.py
def owner_map_uri(self, id: UUID | str) -> str:
    """Where revision `id`'s owner map is materialised.

    Notes
    -----
    - [materialised node caches](https://energy-models.github.io/datarecord/reference/resolution/#materialised-revisions)
    """
    return f"{self.resolved_dir(id)}owner_map.parquet"

dim_owners_uri

dim_owners_uri(id: UUID | str) -> str

Where revision id's dim owners are materialised.

Notes
Source code in src/datarecord/store/store.py
def dim_owners_uri(self, id: UUID | str) -> str:
    """Where revision `id`'s dim owners are materialised.

    Notes
    -----
    - [materialised node caches](https://energy-models.github.io/datarecord/reference/resolution/#materialised-revisions)
    """
    return f"{self.resolved_dir(id)}dim_owners.parquet"

claim

claim(
    *, parent: Revision | None, meta: Meta | None = None
) -> Claim

Hand out a new layer id under parent, by its create-only claim.json.

The layer is pending, reached by no read, until Claim.commit writes its manifest; meta is the revision's, which the commit may extend but not change.

Raises:

Type Description
ValueError

If parent is a revision of another store, naming both locations.

ValidationError

If meta is not flat, each value a string, finite number or boolean.

Notes
Source code in src/datarecord/store/store.py
def claim(self, *, parent: Revision | None, meta: Meta | None = None) -> Claim:
    """Hand out a new layer id under `parent`, by its create-only `claim.json`.

    The layer is pending, reached by no read, until `Claim.commit` writes
    its manifest; `meta` is the revision's, which the commit may extend
    but not change.

    Raises
    ------
    ValueError
        If `parent` is a revision of another store, naming both locations.
    pydantic.ValidationError
        If `meta` is not flat, each value a string, finite number or boolean.

    Notes
    -----
    - [a revision exists only by its file](https://energy-models.github.io/datarecord/reference/store/#a-revision-exists-only-by-its-manifest)
    """
    if parent is not None and parent.store.uri != self.uri:
        msg = (
            f"revision {parent.id} is of the store at {parent.store.uri}, so "
            f"it is no parent in the store at {self.uri}"
        )
        raise ValueError(msg)
    claimed = ClaimFile(
        id=uuid4(),
        parent=None if parent is None else parent.id,
        meta=validate_meta(meta),
        created=datetime.now(UTC),
    )
    claimed.write(self.files, self.claim_uri(claimed.id))
    return Claim(store=self, **asdict(claimed))

commit

commit(
    source: WorkingRecord, *, meta: Meta | None = None
) -> Revision
commit(
    source: LayerData | RecordLike,
    *,
    parent: Revision | None,
    meta: Meta | None = None,
) -> Revision
commit(
    source: LayerData | RecordLike,
    *,
    parent: Revision | None | _Base = OF_WORKING_RECORD,
    meta: Meta | None = None,
) -> Revision

Write source as a new revision under parent: a claim and its commit in one.

A WorkingRecord brings its parent, the revision it was built over, and is written as its edits, a delta against that revision. Any other source names its parent, None for a new root, and is written whole.

Raises:

Type Description
TypeError

If a WorkingRecord comes with a parent, or any other source without one.

ValueError

If w's base is no revision of this store, such as a standalone directory; as claim and Claim.commit do.

Notes
Source code in src/datarecord/store/store.py
def commit(
    self,
    source: LayerData | RecordLike,
    *,
    parent: Revision | None | _Base = _Base.OF_WORKING_RECORD,
    meta: Meta | None = None,
) -> Revision:
    """Write `source` as a new revision under `parent`: a claim and its commit in one.

    A `WorkingRecord` brings its parent, the revision it was built over,
    and is written as its edits, a delta against that revision. Any other
    source names its `parent`, `None` for a new root, and is written
    whole.

    Raises
    ------
    TypeError
        If a `WorkingRecord` comes with a `parent`, or any other source
        without one.
    ValueError
        If `w`'s base is no revision of this store, such as a standalone
        directory; as `claim` and `Claim.commit` do.

    Notes
    -----
    - [writing a whole record](https://energy-models.github.io/datarecord/reference/layers/)
    - [edits](https://energy-models.github.io/datarecord/reference/editing/#edits)
    """
    if isinstance(source, WorkingRecord):
        if parent is not _Base.OF_WORKING_RECORD:
            msg = (
                "a `WorkingRecord` is written over the revision it was built "
                "over, so it takes no `parent`"
            )
            raise TypeError(msg)
        parent = self._base_of(source)
    elif parent is _Base.OF_WORKING_RECORD:
        msg = "`commit` needs a `parent=`, `None` for a new root"
        raise TypeError(msg)
    return self.claim(parent=parent, meta=meta).commit(source)

revision

revision(id: UUID) -> Revision

Look one revision up, reading its manifest and any parents' not yet held.

Raises:

Type Description
KeyError

If this store has no revision id, a pending layer included.

Notes
Source code in src/datarecord/store/store.py
def revision(self, id: UUID) -> Revision:
    """Look one revision up, reading its manifest and any parents' not yet held.

    Raises
    ------
    KeyError
        If this store has no revision `id`, a pending layer included.

    Notes
    -----
    - [the layers](https://energy-models.github.io/datarecord/reference/store/)
    """
    self._cache_upward(id, {})
    return Revision(id=id, store=self)

revisions

revisions(*, meta: Meta | None = None) -> list[Revision]

Every revision in this store whose meta holds each pair of meta, unordered.

A pending layer is none. One listing of layers/, reading only the manifests the workspace does not already hold; a value matches only one of its own type, so 1 matches neither 1.0 nor True.

Raises:

Type Description
ValidationError

If meta is not flat, each value a string, finite number or boolean.

Notes
Source code in src/datarecord/store/store.py
def revisions(self, *, meta: Meta | None = None) -> list[Revision]:
    """Every revision in this store whose `meta` holds each pair of `meta`, unordered.

    A pending layer is none. One listing of `layers/`, reading only the
    manifests the workspace does not already hold; a value matches only
    one of its own type, so `1` matches neither `1.0` nor `True`.

    Raises
    ------
    pydantic.ValidationError
        If `meta` is not flat, each value a string, finite number or boolean.

    Notes
    -----
    - [the layers](https://energy-models.github.io/datarecord/reference/store/)
    """
    wanted = json.dumps(validate_meta(meta))
    self._scan()
    rows = self.workspace.con.execute(
        _in(
            self.workspace,
            "SELECT id FROM {revisions} WHERE location = ? AND json_contains(meta, ?)",
        ),
        [self.uri, wanted],
    ).fetchall()
    return [Revision(id=id, store=self) for (id,) in rows]

claims

claims() -> list[Claim]

Every pending layer in this store, claimed and not committed, oldest first.

Read from the claim files on each call and never cached, since a pending layer may commit at any time.

Notes
Source code in src/datarecord/store/store.py
def claims(self) -> list[Claim]:
    """Every pending layer in this store, claimed and not committed, oldest first.

    Read from the claim files on each call and never cached, since a
    pending layer may commit at any time.

    Notes
    -----
    - [cleanup](https://energy-models.github.io/datarecord/reference/store/#cleanup)
    """
    claims = [Claim(store=self, **asdict(c)) for c in self._claimed()]
    return sorted(claims, key=lambda c: c.created)

cleanup

cleanup(
    *,
    ids: Iterable[UUID] | None = None,
    older_than: timedelta | None = None,
) -> list[UUID]

Delete pending layers: those in ids, or those claimed older_than ago or earlier.

Only a pending layer is deleted: each layer's manifest is rechecked right before its delete, without a lock. A job may hold its claim for days, so nothing deletes but this call, and a layer whose job still runs must be neither named nor young enough to match.

Returns:

Type Description
list of UUID

The ids of the layers deleted.

Raises:

Type Description
TypeError

Unless exactly one of ids and older_than is given.

ValueError

Naming each of ids that is no pending layer, before anything is deleted.

Notes
Source code in src/datarecord/store/store.py
def cleanup(
    self,
    *,
    ids: Iterable[UUID] | None = None,
    older_than: timedelta | None = None,
) -> list[UUID]:
    """Delete pending layers: those in `ids`, or those claimed `older_than` ago or earlier.

    Only a pending layer is deleted: each layer's manifest is rechecked
    right before its delete, without a lock. A job may hold its claim for
    days, so nothing deletes but this call, and a layer whose job still
    runs must be neither named nor young enough to match.

    Returns
    -------
    list of UUID
        The ids of the layers deleted.

    Raises
    ------
    TypeError
        Unless exactly one of `ids` and `older_than` is given.
    ValueError
        Naming each of `ids` that is no pending layer, before anything is
        deleted.

    Notes
    -----
    - [cleanup](https://energy-models.github.io/datarecord/reference/store/#cleanup)
    """
    match ids, older_than:
        case None, timedelta():
            cutoff = datetime.now(UTC) - older_than
            doomed = [c.id for c in self._claimed() if c.created <= cutoff]
        case Iterable(), None:
            doomed = list(dict.fromkeys(ids))
            pending = self._scan()
            refused = [str(i) for i in doomed if i not in pending]
            if refused:
                msg = (
                    f"{refused} are no pending layers of {self.uri}; cleanup "
                    f"deletes only a layer claimed and not committed"
                )
                raise ValueError(msg)
        case _:
            msg = "`cleanup` takes exactly one of `ids=` and `older_than=`"
            raise TypeError(msg)
    deleted = []
    for id in doomed:
        if self.files.exists(self.manifest_uri(id)):
            continue
        self.files.delete(self.layer_dir(id))
        deleted.append(id)
    return deleted

layer

layer(id: UUID) -> Record

Revision id's layer read alone, as the one layer it is.

Under its own manifest, as Workspace.at reads a standalone directory. Not the revision's record - nothing beneath it is folded in.

Raises:

Type Description
KeyError

If this store has no revision id.

Source code in src/datarecord/store/store.py
def layer(self, id: UUID) -> Record:
    """Revision `id`'s layer read alone, as the one layer it is.

    Under its own manifest, as `Workspace.at` reads a standalone directory.
    Not the revision's record - nothing beneath it is folded in.

    Raises
    ------
    KeyError
        If this store has no revision `id`.
    """
    manifest = self.revision(id).manifest
    # Not `id`, whose fold tables hold the whole ancestry's fold.
    return _one_layer(
        self.layer_dir(id), manifest, self.workspace, uuid5(id, "layer")
    )

Claim

Bases: _OfStore

A layer id handed out by Store.claim, whose layer is pending until commit.

Serialises as its store's location and its id, and validates or unpickles back through Workspace.process() by reading its claim file, so a job in another process commits it.

Attributes:

Name Type Description
parent UUID | None

The revision the layer will fold over, None for a root.

meta Meta

The revision's meta as claimed, which the commit keeps.

created datetime

When the id was handed out.

Notes

commit

commit(
    source: LayerData | RecordLike,
    *,
    meta: Meta | None = None,
) -> Revision

Write source as this claim's layer, then its manifest, which makes it a revision.

The data goes straight into the layer directory; the manifest, written create-only and last, lists its files, its full schema, the claim's parent and the claim's meta with meta added. A WorkingRecord is written as its edits.

Raises:

Type Description
ValueError

If meta changes a key the claim set, source is edits staged over another revision than the claim's parent, the source's schema would make the parent's layers unreadable, the layer is committed already, or source fails validation. The layer is left pending.

ValueError

If Store.cleanup deleted the layer, before or while its data was written.

PreconditionFailedError

If another commit of this claim wrote the manifest first.

KeyError

If the claim's parent is gone from the store.

ValidationError

If meta is not flat, each value a string, finite number or boolean.

Notes
Source code in src/datarecord/store/store.py
def commit(
    self, source: LayerData | RecordLike, *, meta: Meta | None = None
) -> Revision:
    """Write `source` as this claim's layer, then its manifest, which makes it a revision.

    The data goes straight into the layer directory; the manifest, written
    create-only and last, lists its files, its full schema, the claim's
    parent and the claim's `meta` with `meta` added. A `WorkingRecord` is
    written as its edits.

    Raises
    ------
    ValueError
        If `meta` changes a key the claim set, `source` is edits staged
        over another revision than the claim's parent, the source's schema
        would make the parent's layers unreadable, the layer is committed
        already, or `source` fails validation. The layer is left pending.
    ValueError
        If `Store.cleanup` deleted the layer, before or while its data was
        written.
    PreconditionFailedError
        If another commit of this claim wrote the manifest first.
    KeyError
        If the claim's parent is gone from the store.
    pydantic.ValidationError
        If `meta` is not flat, each value a string, finite number or boolean.

    Notes
    -----
    - [a revision exists only by its file](https://energy-models.github.io/datarecord/reference/store/#a-revision-exists-only-by-its-manifest)
    """
    store = self.store
    meta = merge_meta(self.meta, validate_meta(meta))
    if isinstance(source, WorkingRecord):
        source = source.edits
    if isinstance(source, StagedSource) and source.base != self.parent:
        msg = (
            f"these edits were staged over revision {source.base}, so they "
            f"are a delta against it and not against {self.parent}; commit "
            f"them as a child of the revision the `WorkingRecord` was built over"
        )
        raise ValueError(msg)
    if store.files.exists(store.manifest_uri(self.id)):
        msg = f"revision {self.id} is committed already; a claim commits once"
        raise ValueError(msg)
    self._refuse_cleaned_up()
    parent = None if self.parent is None else store.revision(self.parent)
    data = layer_data(source, store.workspace.con)
    schema = data.schema
    if parent is not None:
        problems = schema.compatible_with(parent.manifest.schema)
        if problems:
            msg = (
                f"this layer's schema is incompatible with its parent's: "
                f"{'; '.join(problems)} (https://energy-models.github.io/datarecord/reference/schema/#compatibility)"
            )
            raise ValueError(msg)
    with nullcontext() if parent is None else parent.record as base:
        files = write_layer(
            data, store.layer_dir(self.id), store.workspace, parent=base
        )
    manifest = Manifest(
        id=self.id, parent=self.parent, schema=schema, files=files, meta=meta
    )
    self._refuse_cleaned_up()
    manifest.write(store.files, store.manifest_uri(self.id))
    store._cache([manifest])
    return Revision(id=self.id, store=store)

Revision

Bases: _OfStore

A node in the tree of layers.

Each revision adds one parquet layer on top of its parent's; the record it resolves to is that layer over its ancestors'. The layer location derives from id via its store's layer_dir, it is not stored.

Serialises its store as the location, and validates or unpickles back through Workspace.process(), reading the manifest and every parent's the process does not yet hold - so ancestry() answers in a fresh process. Nothing cached travels.

Notes

record property

record: Record

This revision's resolved view, as a Record built once from its ancestry.

The ancestry is truncated at the deepest materialised ancestor, so a deep tree resolves from few entries. Cached rather than rebuilt per call: layers are write-once, and a record built before materialise reads the same answers over the longer ancestry. The Record holds no cursor, so the cache serves every thread. Once it is closed, the next call builds and caches another.

Notes

manifest property

manifest: Manifest

This revision's manifest: its parent, schema, files and meta.

Notes

parent property

parent: UUID | None

This revision's parent's id, as its manifest holds it.

meta property

meta: Meta

This revision's own meta, as its manifest holds it.

materialise

materialise() -> None

Write this node's caches - owner map, dim owners and dead labels.

A policy rather than a lifecycle step, and purely additive: it changes no answer, only how many layers a descendant's read touches. Once these exist, a read stops here instead of walking further up.

Notes
Source code in src/datarecord/store/store.py
def materialise(self) -> None:
    """Write this node's caches - owner map, dim owners and dead labels.

    A policy rather than a lifecycle step, and purely additive: it changes
    no answer, only how many layers a descendant's read touches. Once these
    exist, a read stops here instead of walking further up.

    Notes
    -----
    - [materialised node caches](https://energy-models.github.io/datarecord/reference/resolution/#materialised-revisions)
    """
    materialise(self.id, self.record.sources, self.store)

ancestry

ancestry() -> list[UUID]

Revision ids along the root->self path, root first.

Notes
Source code in src/datarecord/store/store.py
def ancestry(self) -> list[UUID]:
    """Revision ids along the root->self path, root first.

    Notes
    -----
    - [materialised node caches](https://energy-models.github.io/datarecord/reference/resolution/#materialised-revisions)
    """
    return ancestry(self.store, self.id)

Manifest

A layer's manifest.json: written last and create-only, it makes the revision exist.

Attributes:

Name Type Description
parent UUID | None

The revision this layer folds over, None for a root or a standalone directory.

schema InstanceOf[Schema]

The full schema the layer was written under, never a delta. Taken as built where it is handed in, and parsed by read once per distinct schema.

meta Meta

The revision's own: set at the write, and not inherited by a child, unlike schema.meta.

Notes

write

write(files: Files, uri: str) -> None

Write this manifest at uri, create-only.

Raises:

Type Description
PreconditionFailedError

If a manifest is already there.

Source code in src/datarecord/store/manifest.py
def write(self, files: Files, uri: str) -> None:
    """Write this manifest at `uri`, create-only.

    Raises
    ------
    PreconditionFailedError
        If a manifest is already there.
    """
    _create(files, uri, _MANIFEST, self)

read classmethod

read(
    files: Files, uri: str, schemas: dict[str, Schema]
) -> Manifest | None

The manifest at uri, or None where there is none.

Its schema is taken from schemas by schema_digest, and parsed and added there only where it is missing.

Raises:

Type Description
ValidationError

If the manifest, or a schema not in schemas, is malformed.

Source code in src/datarecord/store/manifest.py
@classmethod
def read(
    cls, files: Files, uri: str, schemas: dict[str, Schema]
) -> Manifest | None:
    """The manifest at `uri`, or `None` where there is none.

    Its schema is taken from `schemas` by `schema_digest`, and parsed and
    added there only where it is missing.

    Raises
    ------
    pydantic.ValidationError
        If the manifest, or a schema not in `schemas`, is malformed.
    """
    found = files.read_text(uri)
    if found is None:
        return None
    raw = json.loads(found)
    text, digest = schema_digest(raw["schema"])
    if digest not in schemas:
        schemas[digest] = Schema.model_validate_json(text)
    return _MANIFEST.validate_python({**raw, "schema": schemas[digest]})