Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 11 additions & 4 deletions openrag/components/indexer/indexer.py
Original file line number Diff line number Diff line change
Expand Up @@ -203,10 +203,17 @@ async def update_file_metadata(

# Restore workspace memberships that existed before the delete.
if workspace_ids:
await asyncio.gather(
*[vectordb.add_files_to_workspace.remote(ws_id, [file_id]) for ws_id in workspace_ids]
)
log.debug("Restored workspace memberships after metadata update.", workspace_ids=workspace_ids)
try:
await asyncio.gather(
*[vectordb.add_files_to_workspace.remote(ws_id, [file_id]) for ws_id in workspace_ids]
)
log.debug("Restored workspace memberships after metadata update.", workspace_ids=workspace_ids)
except Exception as ws_err:
log.warning(
"Failed to restore workspace memberships; file is indexed but workspace links may be incomplete.",
error=str(ws_err),
workspace_ids=workspace_ids,
)

log.info("Metadata updated for file.")
except Exception as e:
Expand Down
2 changes: 1 addition & 1 deletion openrag/components/indexer/vectordb/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,6 @@ class WorkspaceFile(Base):
nullable=False,
index=True,
)
file_id = Column(String, nullable=False, index=True)
file_id = Column(Integer, ForeignKey("files.id", ondelete="CASCADE"), nullable=False, index=True)

__table_args__ = (UniqueConstraint("workspace_id", "file_id", name="uix_workspace_file"),)
86 changes: 61 additions & 25 deletions openrag/components/indexer/vectordb/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -634,33 +634,22 @@ def get_workspace(self, workspace_id: str) -> dict | None:
}

def delete_workspace(self, workspace_id: str) -> list[str]:
"""Delete workspace, return list of orphaned file_ids (files only in this workspace).
"""Delete workspace, return list of orphaned file_ids.

A file is only considered orphaned (and eligible for deletion) if it:
- exists in this workspace, AND
- does not appear in any other workspace, AND
- was not independently indexed into the partition (i.e. not in the files table)
A file is orphaned if it belongs to this workspace and no other.
Since WorkspaceFile.file_id is now an integer FK to files.id, every
workspace file has a backing files row, so the only condition is
"not present in any other workspace".
"""
with self.Session() as session:
# Fetch the workspace's partition so we can scope the indexed-files check correctly.
workspace = session.execute(
select(Workspace).where(Workspace.workspace_id == workspace_id)
).scalar_one_or_none()
if workspace is None:
return []
partition = workspace.partition_name

# Files present in at least one other workspace
# File PKs present in at least one other workspace
subq_other_ws = select(WorkspaceFile.file_id).where(WorkspaceFile.workspace_id != workspace_id)
# Files that were independently indexed in the same partition.
# Scoped to the partition so a same-named file in another partition
# does not incorrectly block orphan detection here.
subq_indexed = select(File.file_id).where(File.partition_name == partition)
# Orphaned = in this workspace, not in any other
result = session.execute(
select(WorkspaceFile.file_id)
select(File.file_id)
.join(WorkspaceFile, WorkspaceFile.file_id == File.id)
.where(WorkspaceFile.workspace_id == workspace_id)
.where(WorkspaceFile.file_id.notin_(subq_other_ws))
.where(WorkspaceFile.file_id.notin_(subq_indexed))
)
orphaned_file_ids = [r[0] for r in result.all()]

Expand All @@ -669,38 +658,80 @@ def delete_workspace(self, workspace_id: str) -> list[str]:
session.commit()
return orphaned_file_ids

def get_existing_file_ids(self, partition: str, file_ids: list[str]) -> set[str]:
"""Return the subset of *file_ids* that actually exist in *partition*."""
with self.Session() as session:
result = session.execute(
select(File.file_id).where(
File.partition_name == partition,
File.file_id.in_(file_ids),
)
)
return {r[0] for r in result.all()}

def add_files_to_workspace(self, workspace_id: str, file_ids: list[str]):
with self.Session() as session:
# Resolve the workspace's partition to scope the File lookup
workspace = session.execute(
select(Workspace).where(Workspace.workspace_id == workspace_id)
).scalar_one_or_none()
if workspace is None:
return
partition = workspace.partition_name
for fid in file_ids:
stmt = pg_insert(WorkspaceFile).values(workspace_id=workspace_id, file_id=fid)
file_row = session.execute(
select(File.id).where(File.file_id == fid, File.partition_name == partition)
).scalar_one_or_none()
if file_row is None:
continue
stmt = pg_insert(WorkspaceFile).values(workspace_id=workspace_id, file_id=file_row)
stmt = stmt.on_conflict_do_nothing(constraint="uix_workspace_file")
session.execute(stmt)
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
session.commit()

def remove_file_from_workspace(self, workspace_id: str, file_id: str) -> bool:
"""Remove a file from a workspace. Returns True if the association existed, False otherwise."""
with self.Session() as session:
workspace = session.execute(
select(Workspace).where(Workspace.workspace_id == workspace_id)
).scalar_one_or_none()
if workspace is None:
return False
file_pk = session.execute(
select(File.id).where(File.file_id == file_id, File.partition_name == workspace.partition_name)
).scalar_one_or_none()
if file_pk is None:
return False
result = session.execute(
delete(WorkspaceFile).where(
WorkspaceFile.workspace_id == workspace_id,
WorkspaceFile.file_id == file_id,
WorkspaceFile.file_id == file_pk,
)
)
session.commit()
return result.rowcount > 0

def list_workspace_files(self, workspace_id: str) -> list[str]:
with self.Session() as session:
result = session.execute(select(WorkspaceFile.file_id).where(WorkspaceFile.workspace_id == workspace_id))
result = session.execute(
select(File.file_id)
.join(WorkspaceFile, WorkspaceFile.file_id == File.id)
.where(WorkspaceFile.workspace_id == workspace_id)
)
return [r[0] for r in result.all()]

def get_file_workspaces(self, file_id: str, partition: str) -> list[str]:
"""Return the workspace IDs that contain the given file, scoped to the given partition."""
with self.Session() as session:
file_pk = session.execute(
select(File.id).where(File.file_id == file_id, File.partition_name == partition)
).scalar_one_or_none()
if file_pk is None:
return []
ws_ids = select(Workspace.workspace_id).where(Workspace.partition_name == partition)
result = session.execute(
select(WorkspaceFile.workspace_id).where(
WorkspaceFile.file_id == file_id,
WorkspaceFile.file_id == file_pk,
WorkspaceFile.workspace_id.in_(ws_ids),
)
)
Expand All @@ -709,10 +740,15 @@ def get_file_workspaces(self, file_id: str, partition: str) -> list[str]:
def remove_file_from_all_workspaces(self, file_id: str, partition: str):
"""Remove file from all workspaces in the given partition — called during file deletion."""
with self.Session() as session:
file_pk = session.execute(
select(File.id).where(File.file_id == file_id, File.partition_name == partition)
).scalar_one_or_none()
if file_pk is None:
return
ws_ids = select(Workspace.workspace_id).where(Workspace.partition_name == partition)
session.execute(
delete(WorkspaceFile).where(
WorkspaceFile.file_id == file_id,
WorkspaceFile.file_id == file_pk,
WorkspaceFile.workspace_id.in_(ws_ids),
)
)
Expand Down
4 changes: 4 additions & 0 deletions openrag/components/indexer/vectordb/vectordb.py
Original file line number Diff line number Diff line change
Expand Up @@ -1123,6 +1123,10 @@ async def delete_workspace(self, workspace_id: str) -> list[str]:
"""Delete workspace and return orphaned file_ids. Caller must delete those files from Milvus."""
return self.partition_file_manager.delete_workspace(workspace_id)

async def get_existing_file_ids(self, partition: str, file_ids: list[str]) -> list[str]:
"""Return the subset of file_ids that exist in the given partition."""
return list(self.partition_file_manager.get_existing_file_ids(partition, file_ids))

async def add_files_to_workspace(self, workspace_id: str, file_ids: list[str]):
self.partition_file_manager.add_files_to_workspace(workspace_id, file_ids)

Expand Down
7 changes: 6 additions & 1 deletion openrag/routers/indexer.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
# load config
config = load_config()
DATA_DIR = config.paths.data_dir
VECTORDB_TIMEOUT = config.ray.indexer.get("vectordb_timeout", 30)

FORBIDDEN_CHARS_IN_FILE_ID = set("/") # set('"<>#%{}|\\^`[]')
LOG_FILE = Path(config.paths.log_dir or "logs") / "app.json"
Expand Down Expand Up @@ -272,7 +273,11 @@ async def put_file(
)

# Snapshot workspace memberships before deletion so they can be restored on the new version.
existing_workspace_ids = await vectordb.get_file_workspaces.remote(file_id, partition)
existing_workspace_ids = await call_ray_actor_with_timeout(
future=vectordb.get_file_workspaces.remote(file_id, partition),
timeout=VECTORDB_TIMEOUT,
task_description=f"get_file_workspaces({file_id}, {partition})",
)

# Delete the existing file from the vector database
await indexer.delete_file.remote(file_id, partition)
Expand Down
12 changes: 12 additions & 0 deletions openrag/routers/workspaces.py
Original file line number Diff line number Diff line change
Expand Up @@ -153,11 +153,23 @@ async def delete_workspace(
dependencies=[Depends(require_partition_editor)],
)
async def add_files_to_workspace(
partition: str,
workspace_id: str,
body: AddFilesRequest,
vectordb=Depends(get_vectordb),
_ws=Depends(require_workspace_in_partition),
):
existing_ids = await call_ray_actor_with_timeout(
vectordb.get_existing_file_ids.remote(partition, body.file_ids),
timeout=VECTORDB_TIMEOUT,
task_description=f"get_existing_file_ids({partition})",
)
unknown_ids = sorted(set(body.file_ids) - set(existing_ids))
if unknown_ids:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"File IDs not found in partition '{partition}': {unknown_ids}",
)
await call_ray_actor_with_timeout(
vectordb.add_files_to_workspace.remote(workspace_id, body.file_ids),
timeout=VECTORDB_TIMEOUT,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
"""add FK from workspace_files.file_id (int) to files.id

Revision ID: f1a2b3c4d5e6
Revises: e7f8a9b0c1d2
Create Date: 2026-03-12 10:00:00.000000

"""

from collections.abc import Sequence

import sqlalchemy as sa
from alembic import op

# revision identifiers, used by Alembic.
revision: str = "f1a2b3c4d5e6"
down_revision: str | Sequence[str] | None = "e7f8a9b0c1d2"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None


def upgrade() -> None:
"""Migrate workspace_files.file_id from string to integer FK referencing files.id."""
# 1. Purge rows that have no matching file (no valid files.file_id to JOIN against).
op.execute("DELETE FROM workspace_files WHERE file_id NOT IN (SELECT file_id FROM files)")

# 2. Add a temporary integer column to hold the resolved files.id value.
op.add_column("workspace_files", sa.Column("file_fk", sa.Integer(), nullable=True))

# 3. Populate it by joining on the string file_id, scoped to the workspace's partition
# to resolve ambiguity when the same filename exists in multiple partitions.
op.execute(
"UPDATE workspace_files wf "
"SET file_fk = f.id "
"FROM files f "
"JOIN workspaces w ON w.workspace_id = wf.workspace_id "
"WHERE f.file_id = wf.file_id AND f.partition_name = w.partition_name"
)

# 3b. Drop any rows that couldn't be resolved (file_fk still NULL).
op.execute("DELETE FROM workspace_files WHERE file_fk IS NULL")

# 4. Drop the old string column and its index.
op.drop_index("ix_workspace_files_file_id", table_name="workspace_files")
op.drop_column("workspace_files", "file_id")

# 5. Rename file_fk → file_id, make it NOT NULL.
op.alter_column("workspace_files", "file_fk", new_column_name="file_id", nullable=False)

# 6. Recreate the index, unique constraint, and FK.
op.create_index("ix_workspace_files_file_id", "workspace_files", ["file_id"])
op.create_unique_constraint("uix_workspace_file", "workspace_files", ["workspace_id", "file_id"])
op.create_foreign_key(
"fk_workspace_files_file_id",
"workspace_files",
"files",
["file_id"],
["id"],
ondelete="CASCADE",
)
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved


def downgrade() -> None:
"""Revert workspace_files.file_id back to a string column."""
op.drop_constraint("fk_workspace_files_file_id", "workspace_files", type_="foreignkey")
op.drop_index("ix_workspace_files_file_id", table_name="workspace_files")

# Re-add a string column and repopulate from files.file_id via JOIN.
op.add_column("workspace_files", sa.Column("file_str", sa.String(), nullable=True))
op.execute("UPDATE workspace_files wf SET file_str = f.file_id FROM files f WHERE f.id = wf.file_id")
op.drop_column("workspace_files", "file_id")
op.alter_column("workspace_files", "file_str", new_column_name="file_id", nullable=False)
op.create_index("ix_workspace_files_file_id", "workspace_files", ["file_id"])
op.create_unique_constraint("uix_workspace_file", "workspace_files", ["workspace_id", "file_id"])
17 changes: 17 additions & 0 deletions tests/api_tests/test_workspaces.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,20 @@ def test_delete_workspace(self, api_client, workspace_partition, workspace_id):


class TestWorkspaceFiles:
@staticmethod
def _upload_file(api_client, partition: str, file_id: str, content: str = "Test content"):
file_obj = io.BytesIO(content.encode())
response = api_client.post(
f"/indexer/partition/{partition}/file/{file_id}",
files={"file": (f"{file_id}.txt", file_obj, "text/plain")},
data={"metadata": "{}"},
)
assert response.status_code in [200, 201, 202]
wait_for_indexing(api_client, response.json())

def test_add_files_to_workspace(self, api_client, workspace_partition, workspace_id):
self._upload_file(api_client, workspace_partition, "file-a")
self._upload_file(api_client, workspace_partition, "file-b")
api_client.post(
f"/partition/{workspace_partition}/workspaces",
json={"workspace_id": workspace_id},
Expand All @@ -89,6 +102,7 @@ def test_add_files_to_workspace(self, api_client, workspace_partition, workspace
assert set(response.json()["file_ids"]) == {"file-a", "file-b"}

def test_list_workspace_files(self, api_client, workspace_partition, workspace_id):
self._upload_file(api_client, workspace_partition, "file-a")
api_client.post(
f"/partition/{workspace_partition}/workspaces",
json={"workspace_id": workspace_id},
Expand All @@ -102,6 +116,8 @@ def test_list_workspace_files(self, api_client, workspace_partition, workspace_i
assert "file-a" in response.json()["file_ids"]

def test_remove_file_from_workspace(self, api_client, workspace_partition, workspace_id):
self._upload_file(api_client, workspace_partition, "file-a")
self._upload_file(api_client, workspace_partition, "file-b")
api_client.post(
f"/partition/{workspace_partition}/workspaces",
json={"workspace_id": workspace_id},
Expand All @@ -119,6 +135,7 @@ def test_remove_file_from_workspace(self, api_client, workspace_partition, works
assert "file-b" in files_resp.json()["file_ids"]

def test_add_same_file_to_multiple_workspaces(self, api_client, workspace_partition):
self._upload_file(api_client, workspace_partition, "shared-file")
ws1 = f"ws-{uuid.uuid4().hex[:8]}"
ws2 = f"ws-{uuid.uuid4().hex[:8]}"
api_client.post(f"/partition/{workspace_partition}/workspaces", json={"workspace_id": ws1})
Expand Down
Loading