Skip to content

Commit

Permalink
Deployed d271df4 to v0.10.0 with MkDocs 1.6.0 and mike 2.1.2
Browse files Browse the repository at this point in the history
  • Loading branch information
github-actions[bot] committed Jul 17, 2024
1 parent 048105c commit 00f98ff
Show file tree
Hide file tree
Showing 146 changed files with 38,366 additions and 3 deletions.
2 changes: 1 addition & 1 deletion latest
860 changes: 860 additions & 0 deletions v0.10.0/404.html

Large diffs are not rendered by default.

1,083 changes: 1,083 additions & 0 deletions v0.10.0/CONTRIBUTING/index.html

Large diffs are not rendered by default.

13 changes: 13 additions & 0 deletions v0.10.0/_code/duckdb_example.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
import duckdb

from lakefs_spec import LakeFSFileSystem

fs = LakeFSFileSystem()
duckdb.register_filesystem(fs)

with fs.transaction("quickstart", "main") as tx:
lakes = duckdb.read_parquet("lakefs://quickstart/main/lakes.parquet")
italian_lakes = duckdb.sql("SELECT * FROM lakes where Country='Italy'")
italian_lakes.to_csv(f"lakefs://quickstart/{tx.branch.id}/italian_lakes.csv")

tx.commit(message="Add Italian lakes")
12 changes: 12 additions & 0 deletions v0.10.0/_code/hf_datasets_example.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
from datasets import load_dataset

from lakefs_spec import LakeFSFileSystem

fs = LakeFSFileSystem()

with fs.transaction("quickstart", "main") as tx:
lakes = load_dataset("parquet", data_files="lakefs://quickstart/main/lakes.parquet")
irish_lakes = lakes.filter(lambda lake: lake["Country"] == "Ireland")
irish_lakes.save_to_disk(f"lakefs://quickstart/{tx.branch.id}/irish_lakes")

tx.commit(message="Add Irish lakes dataset")
12 changes: 12 additions & 0 deletions v0.10.0/_code/pandas_example.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
import pandas as pd

from lakefs_spec import LakeFSFileSystem

fs = LakeFSFileSystem()

with fs.transaction("quickstart", "main") as tx:
lakes = pd.read_parquet(f"lakefs://quickstart/{tx.branch.id}/lakes.parquet")
german_lakes = lakes.query('Country == "Germany"')
german_lakes.to_csv(f"lakefs://quickstart/{tx.branch.id}/german_lakes.csv")

tx.commit(message="Add German lakes")
14 changes: 14 additions & 0 deletions v0.10.0/_code/polars_example.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
import polars as pl

from lakefs_spec import LakeFSFileSystem

fs = LakeFSFileSystem()

with fs.transaction("quickstart", "main") as tx:
lakes = pl.read_parquet(f"lakefs://quickstart/{tx.branch.id}/lakes.parquet")
us_lakes = lakes.filter(pl.col("Country") == "United States of America")

with fs.open(f"lakefs://quickstart/{tx.branch.id}/us_lakes.csv", "wb") as f:
us_lakes.write_csv(f) # (1)!

tx.commit(message="Add US lakes")
20 changes: 20 additions & 0 deletions v0.10.0/_code/pyarrow_example.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
import pyarrow as pa
import pyarrow.dataset as ds
import pyarrow.parquet as pq

from lakefs_spec import LakeFSFileSystem

fs = LakeFSFileSystem()

with fs.transaction("quickstart", "main") as tx:
lakes_table = pq.read_table(f"quickstart/{tx.branch.id}/lakes.parquet", filesystem=fs)

ds.write_dataset(
lakes_table,
f"quickstart/{tx.branch.id}/lakes",
filesystem=fs,
format="csv",
partitioning=ds.partitioning(pa.schema([lakes_table.schema.field("Country")])),
)

tx.commit("Add partitioned lakes data set")
37 changes: 37 additions & 0 deletions v0.10.0/_code/quickstart.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
from pathlib import Path

from lakefs_spec import LakeFSFileSystem

REPO, BRANCH = "repo", "main"

# Prepare example local data
local_path = Path("demo.txt")
local_path.write_text("Hello, lakeFS!")

# Upload the local file to the repo and commit
fs = LakeFSFileSystem() # will auto-discover credentials from ~/.lakectl.yaml
repo_path = f"{REPO}/{BRANCH}/{local_path.name}"

with fs.transaction(REPO, BRANCH) as tx:
fs.put(str(local_path), f"{REPO}/{tx.branch.id}/{local_path.name}")
tx.commit(message="Add demo data")

# Read back the file contents
f = fs.open(repo_path, "rt")
print(f.readline()) # prints "Hello, lakeFS!"

# Compare the sizes of local file and repo
file_info = fs.info(repo_path)
print(
f"{local_path.name}: local size: {file_info['size']}, remote size: {local_path.stat().st_size}"
)

# Get information about all files in the repo root
print(fs.ls(f"{REPO}/{BRANCH}/"))

# Delete uploaded file from the repository (and commit)
with fs.transaction(REPO, BRANCH) as tx:
fs.rm(f"{REPO}/{tx.branch.id}/{local_path.name}")
tx.commit(message="Delete demo data")

local_path.unlink()
Binary file added v0.10.0/_images/aai-favicon.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added v0.10.0/_images/aai-logo-cropped.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added v0.10.0/_images/lakefs-spec-favicon.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added v0.10.0/_images/lakefs-spec-logo-dark.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added v0.10.0/_images/lakefs-spec-logo-light.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added v0.10.0/_images/quickstart-lakefs-ui.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
65 changes: 65 additions & 0 deletions v0.10.0/_scripts/gen_api_ref_pages.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
"""Automatically generate API reference pages from source files.
Source: https://mkdocstrings.github.io/recipes/#automatic-code-reference-pages
Note: this script assumes a source layout with a `src/` folder.
"""

import ast
import logging
from pathlib import Path

import docstring_parser
import mkdocs_gen_files

nav = mkdocs_gen_files.Nav()

for path in sorted(Path("src").rglob("*.py")):
module_path = path.relative_to("src").with_suffix("")
doc_path = path.relative_to("src").with_suffix(".md")
full_doc_path = Path("reference", doc_path)

parts = list(module_path.parts)

if parts[-1] == "__init__":
parts = parts[:-1]
doc_path = doc_path.with_name("index.md")
full_doc_path = full_doc_path.with_name("index.md")
elif parts[-1] == "__main__":
continue

nav[parts] = doc_path.as_posix()

with mkdocs_gen_files.open(full_doc_path, "w") as fd:
identifier = ".".join(parts)
print("::: " + identifier, file=fd)

mkdocs_gen_files.set_edit_path(full_doc_path, path)

# Add links for top-level modules to root page
root_page = next(it for it in nav.items() if it.level == 0)
children = [it for it in nav.items() if it.level == 1]

with mkdocs_gen_files.open(f"reference/{root_page.filename}", "a") as f:
f.write("## Modules\n")
for ch in children:
f.write(f"### [{ch.title}](../{ch.filename})\n")

try:
source_file = Path("src", ch.filename).with_suffix(".py")

# Index page for submodules maps to __init__.py of the module
if source_file.stem == "index":
source_file = source_file.with_stem("__init__")

tree = ast.parse(source_file.read_text())
docstring = ast.get_docstring(tree, clean=False)
doc = docstring_parser.parse(docstring)

if doc.short_description:
f.write(f"{doc.short_description}\n\n")
except Exception as e:
logging.warning(f"Could not parse module docstring: {ch.filename}", exc_info=True)

with mkdocs_gen_files.open("reference/SUMMARY.md", "w") as nav_file:
nav_file.writelines(nav.build_literate_nav())
20 changes: 20 additions & 0 deletions v0.10.0/_scripts/jupytext_convert.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
"""Automatically convert Jupytext files to Notebooks"""

import logging
from pathlib import Path

import jupytext
import mkdocs_gen_files

for path in sorted(Path("docs/tutorials").rglob("*.py")):
ipynb_path = path.relative_to(Path("docs")).with_suffix(".ipynb")
logging.debug(f"Converting {path} -> {ipynb_path}")

jupytext_file = path.read_text(encoding="utf-8")
jupytext_nb = jupytext.reads(jupytext_file, fmt="py:percent")

with mkdocs_gen_files.open(ipynb_path, "wb") as fd:
content = jupytext.writes(jupytext_nb, fmt="notebook")
fd.write(content.encode("utf8"))

mkdocs_gen_files.set_edit_path(ipynb_path, path)
46 changes: 46 additions & 0 deletions v0.10.0/_styles/extra.css
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
/*Remove leading `!` from code cells, which use the cell magic for shell commands*/
code>.err:first-of-type {
display: none;
}

/* Hide empty root heading (-> `#`) */
h1#_1 {
display: none;
}

.logo {
/* No wider than the screen on smaller screens */
max-width: min(100%, 100vw) !important;
max-height: 196px;

/* Horizontally centered */
display: block;
margin: 0 auto;
}

table {
display: block;
max-width: -moz-fit-content;
max-width: fit-content;
margin: 0 auto;
overflow-x: auto;
white-space: nowrap;
}

.nt-card .landing-page-icon {
color: #b7c6cf;
}

.nt-card:hover .landing-page-icon {
color: var(--md-accent-fg-color);
}

.celltag_Remove_all_output .output_wrapper,
.celltag_Remove_single_output .output_wrapper {
display: none !important
}


.celltag_Remove_input .input_wrapper {
display: none !important
}
Loading

0 comments on commit 00f98ff

Please sign in to comment.