close
Skip to content

feat: expose base scoped store bindings to python#6547

Open
zhangyue19921010 wants to merge 4 commits intolance-format:mainfrom
zhangyue19921010:base-scoped-store-bindings-python
Open

feat: expose base scoped store bindings to python#6547
zhangyue19921010 wants to merge 4 commits intolance-format:mainfrom
zhangyue19921010:base-scoped-store-bindings-python

Conversation

@zhangyue19921010
Copy link
Copy Markdown
Contributor

@zhangyue19921010 zhangyue19921010 commented Apr 16, 2026

Tested: Python can write to different S3 buckets within the same multi-base dataset, using different storage configs for each base.configs.

from pathlib import Path
import shutil

import lance
import pyarrow as pa
from lance import DatasetBasePath


def main():
    path0 = "s3://ceshi/las/lance/multi-base-write2"
    path1 = "s3://zy-test-lance/zy-test-lance/multi-base-write2/"

    storage_options_0 = {
        "access_key_id": "",
        "secret_access_key": "==",
        "aws_endpoint": "",
        "aws_region": "",
        "virtual_hosted_style_request": "true",
    }

    storage_options_1 = {
        "access_key_id": "",
        "secret_access_key": "==",
        "aws_region": "",
        "aws_endpoint": "",
        "virtual_hosted_style_request": "true"
    }

    base_store_params = {
        path1: storage_options_1,
    }

    local_rows = pa.Table.from_pylist(
        [
            {"name": "Alice", "age": 20},
            {"name": "Jack", "age": 20},
        ]
    )

    s3_rows = pa.Table.from_pylist(
        [
            {"name": "Bob", "age": 30},
        ]
    )

    ds = lance.write_dataset(
        local_rows,
        path0,
        mode="create",
        initial_bases=[
            DatasetBasePath(path0, name="s3", is_dataset_root=True),
            DatasetBasePath(path1, name="s3"),
        ],
        target_bases=["s3"],
        base_store_params=base_store_params,
        storage_options=storage_options_0,
    )

    ds = lance.write_dataset(
        s3_rows,
        ds,
        mode="append",
        target_bases=["s3"],
        base_store_params=base_store_params,
        storage_options=storage_options_0,
    )


if __name__ == "__main__":
    main()


@github-actions github-actions bot added enhancement New feature or request python labels Apr 16, 2026
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Labels

enhancement New feature or request python

Projects

None yet

Development

Successfully merging this pull request may close these issues.

1 participant