-
Notifications
You must be signed in to change notification settings - Fork 4
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Deployed d271df4 to v0.10.0 with MkDocs 1.6.0 and mike 2.1.2
- Loading branch information
1 parent
048105c
commit 00f98ff
Showing
146 changed files
with
38,366 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1 +1 @@ | ||
v0.9.0 | ||
v0.10.0 |
Large diffs are not rendered by default.
Oops, something went wrong.
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,13 @@ | ||
import duckdb | ||
|
||
from lakefs_spec import LakeFSFileSystem | ||
|
||
fs = LakeFSFileSystem() | ||
duckdb.register_filesystem(fs) | ||
|
||
with fs.transaction("quickstart", "main") as tx: | ||
lakes = duckdb.read_parquet("lakefs://quickstart/main/lakes.parquet") | ||
italian_lakes = duckdb.sql("SELECT * FROM lakes where Country='Italy'") | ||
italian_lakes.to_csv(f"lakefs://quickstart/{tx.branch.id}/italian_lakes.csv") | ||
|
||
tx.commit(message="Add Italian lakes") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
from datasets import load_dataset | ||
|
||
from lakefs_spec import LakeFSFileSystem | ||
|
||
fs = LakeFSFileSystem() | ||
|
||
with fs.transaction("quickstart", "main") as tx: | ||
lakes = load_dataset("parquet", data_files="lakefs://quickstart/main/lakes.parquet") | ||
irish_lakes = lakes.filter(lambda lake: lake["Country"] == "Ireland") | ||
irish_lakes.save_to_disk(f"lakefs://quickstart/{tx.branch.id}/irish_lakes") | ||
|
||
tx.commit(message="Add Irish lakes dataset") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
import pandas as pd | ||
|
||
from lakefs_spec import LakeFSFileSystem | ||
|
||
fs = LakeFSFileSystem() | ||
|
||
with fs.transaction("quickstart", "main") as tx: | ||
lakes = pd.read_parquet(f"lakefs://quickstart/{tx.branch.id}/lakes.parquet") | ||
german_lakes = lakes.query('Country == "Germany"') | ||
german_lakes.to_csv(f"lakefs://quickstart/{tx.branch.id}/german_lakes.csv") | ||
|
||
tx.commit(message="Add German lakes") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,14 @@ | ||
import polars as pl | ||
|
||
from lakefs_spec import LakeFSFileSystem | ||
|
||
fs = LakeFSFileSystem() | ||
|
||
with fs.transaction("quickstart", "main") as tx: | ||
lakes = pl.read_parquet(f"lakefs://quickstart/{tx.branch.id}/lakes.parquet") | ||
us_lakes = lakes.filter(pl.col("Country") == "United States of America") | ||
|
||
with fs.open(f"lakefs://quickstart/{tx.branch.id}/us_lakes.csv", "wb") as f: | ||
us_lakes.write_csv(f) # (1)! | ||
|
||
tx.commit(message="Add US lakes") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,20 @@ | ||
import pyarrow as pa | ||
import pyarrow.dataset as ds | ||
import pyarrow.parquet as pq | ||
|
||
from lakefs_spec import LakeFSFileSystem | ||
|
||
fs = LakeFSFileSystem() | ||
|
||
with fs.transaction("quickstart", "main") as tx: | ||
lakes_table = pq.read_table(f"quickstart/{tx.branch.id}/lakes.parquet", filesystem=fs) | ||
|
||
ds.write_dataset( | ||
lakes_table, | ||
f"quickstart/{tx.branch.id}/lakes", | ||
filesystem=fs, | ||
format="csv", | ||
partitioning=ds.partitioning(pa.schema([lakes_table.schema.field("Country")])), | ||
) | ||
|
||
tx.commit("Add partitioned lakes data set") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,37 @@ | ||
from pathlib import Path | ||
|
||
from lakefs_spec import LakeFSFileSystem | ||
|
||
REPO, BRANCH = "repo", "main" | ||
|
||
# Prepare example local data | ||
local_path = Path("demo.txt") | ||
local_path.write_text("Hello, lakeFS!") | ||
|
||
# Upload the local file to the repo and commit | ||
fs = LakeFSFileSystem() # will auto-discover credentials from ~/.lakectl.yaml | ||
repo_path = f"{REPO}/{BRANCH}/{local_path.name}" | ||
|
||
with fs.transaction(REPO, BRANCH) as tx: | ||
fs.put(str(local_path), f"{REPO}/{tx.branch.id}/{local_path.name}") | ||
tx.commit(message="Add demo data") | ||
|
||
# Read back the file contents | ||
f = fs.open(repo_path, "rt") | ||
print(f.readline()) # prints "Hello, lakeFS!" | ||
|
||
# Compare the sizes of local file and repo | ||
file_info = fs.info(repo_path) | ||
print( | ||
f"{local_path.name}: local size: {file_info['size']}, remote size: {local_path.stat().st_size}" | ||
) | ||
|
||
# Get information about all files in the repo root | ||
print(fs.ls(f"{REPO}/{BRANCH}/")) | ||
|
||
# Delete uploaded file from the repository (and commit) | ||
with fs.transaction(REPO, BRANCH) as tx: | ||
fs.rm(f"{REPO}/{tx.branch.id}/{local_path.name}") | ||
tx.commit(message="Delete demo data") | ||
|
||
local_path.unlink() |
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,65 @@ | ||
"""Automatically generate API reference pages from source files. | ||
Source: https://mkdocstrings.github.io/recipes/#automatic-code-reference-pages | ||
Note: this script assumes a source layout with a `src/` folder. | ||
""" | ||
|
||
import ast | ||
import logging | ||
from pathlib import Path | ||
|
||
import docstring_parser | ||
import mkdocs_gen_files | ||
|
||
nav = mkdocs_gen_files.Nav() | ||
|
||
for path in sorted(Path("src").rglob("*.py")): | ||
module_path = path.relative_to("src").with_suffix("") | ||
doc_path = path.relative_to("src").with_suffix(".md") | ||
full_doc_path = Path("reference", doc_path) | ||
|
||
parts = list(module_path.parts) | ||
|
||
if parts[-1] == "__init__": | ||
parts = parts[:-1] | ||
doc_path = doc_path.with_name("index.md") | ||
full_doc_path = full_doc_path.with_name("index.md") | ||
elif parts[-1] == "__main__": | ||
continue | ||
|
||
nav[parts] = doc_path.as_posix() | ||
|
||
with mkdocs_gen_files.open(full_doc_path, "w") as fd: | ||
identifier = ".".join(parts) | ||
print("::: " + identifier, file=fd) | ||
|
||
mkdocs_gen_files.set_edit_path(full_doc_path, path) | ||
|
||
# Add links for top-level modules to root page | ||
root_page = next(it for it in nav.items() if it.level == 0) | ||
children = [it for it in nav.items() if it.level == 1] | ||
|
||
with mkdocs_gen_files.open(f"reference/{root_page.filename}", "a") as f: | ||
f.write("## Modules\n") | ||
for ch in children: | ||
f.write(f"### [{ch.title}](../{ch.filename})\n") | ||
|
||
try: | ||
source_file = Path("src", ch.filename).with_suffix(".py") | ||
|
||
# Index page for submodules maps to __init__.py of the module | ||
if source_file.stem == "index": | ||
source_file = source_file.with_stem("__init__") | ||
|
||
tree = ast.parse(source_file.read_text()) | ||
docstring = ast.get_docstring(tree, clean=False) | ||
doc = docstring_parser.parse(docstring) | ||
|
||
if doc.short_description: | ||
f.write(f"{doc.short_description}\n\n") | ||
except Exception as e: | ||
logging.warning(f"Could not parse module docstring: {ch.filename}", exc_info=True) | ||
|
||
with mkdocs_gen_files.open("reference/SUMMARY.md", "w") as nav_file: | ||
nav_file.writelines(nav.build_literate_nav()) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,20 @@ | ||
"""Automatically convert Jupytext files to Notebooks""" | ||
|
||
import logging | ||
from pathlib import Path | ||
|
||
import jupytext | ||
import mkdocs_gen_files | ||
|
||
for path in sorted(Path("docs/tutorials").rglob("*.py")): | ||
ipynb_path = path.relative_to(Path("docs")).with_suffix(".ipynb") | ||
logging.debug(f"Converting {path} -> {ipynb_path}") | ||
|
||
jupytext_file = path.read_text(encoding="utf-8") | ||
jupytext_nb = jupytext.reads(jupytext_file, fmt="py:percent") | ||
|
||
with mkdocs_gen_files.open(ipynb_path, "wb") as fd: | ||
content = jupytext.writes(jupytext_nb, fmt="notebook") | ||
fd.write(content.encode("utf8")) | ||
|
||
mkdocs_gen_files.set_edit_path(ipynb_path, path) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,46 @@ | ||
/*Remove leading `!` from code cells, which use the cell magic for shell commands*/ | ||
code>.err:first-of-type { | ||
display: none; | ||
} | ||
|
||
/* Hide empty root heading (-> `#`) */ | ||
h1#_1 { | ||
display: none; | ||
} | ||
|
||
.logo { | ||
/* No wider than the screen on smaller screens */ | ||
max-width: min(100%, 100vw) !important; | ||
max-height: 196px; | ||
|
||
/* Horizontally centered */ | ||
display: block; | ||
margin: 0 auto; | ||
} | ||
|
||
table { | ||
display: block; | ||
max-width: -moz-fit-content; | ||
max-width: fit-content; | ||
margin: 0 auto; | ||
overflow-x: auto; | ||
white-space: nowrap; | ||
} | ||
|
||
.nt-card .landing-page-icon { | ||
color: #b7c6cf; | ||
} | ||
|
||
.nt-card:hover .landing-page-icon { | ||
color: var(--md-accent-fg-color); | ||
} | ||
|
||
.celltag_Remove_all_output .output_wrapper, | ||
.celltag_Remove_single_output .output_wrapper { | ||
display: none !important | ||
} | ||
|
||
|
||
.celltag_Remove_input .input_wrapper { | ||
display: none !important | ||
} |
Oops, something went wrong.