Skip to content
Merged
Show file tree
Hide file tree
Changes from 13 commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
e030f39
Add Backup integration
ludeeus Feb 12, 2022
31fff38
Mark handle_remove WS as async
ludeeus Feb 13, 2022
16c8ac7
Allow symlink
ludeeus Feb 13, 2022
549beaa
Remove _is_excluded_by_filter function
ludeeus Feb 13, 2022
373410a
limit to file
ludeeus Feb 13, 2022
705d833
Store directly instead of scan when generating new
ludeeus Feb 13, 2022
2c109d7
wrap in try/finally
ludeeus Feb 13, 2022
f3f65ea
Use debug for logs
ludeeus Feb 13, 2022
3fc7464
other adjustments
ludeeus Feb 13, 2022
75c08b0
Adjust test
ludeeus Feb 13, 2022
6326953
Move dir creation to backup creation
ludeeus Feb 14, 2022
63a48e2
Update homeassistant/components/backup/manager.py
ludeeus Feb 14, 2022
e78ba00
adjust
ludeeus Feb 14, 2022
e759152
Merge branch 'dev' of github.com:home-assistant/core into backup_inte…
ludeeus Feb 21, 2022
1915492
broken
ludeeus Feb 21, 2022
947c097
working again
ludeeus Feb 24, 2022
d90bed0
Merge branch 'dev' of github.com:home-assistant/core into backup_inte…
ludeeus Feb 24, 2022
8bef482
Add to default_config
ludeeus Feb 24, 2022
5918672
Fix hassfest
ludeeus Feb 24, 2022
8a8a555
Load when needed
ludeeus Feb 24, 2022
2858697
error
ludeeus Feb 24, 2022
f69374f
remove
ludeeus Feb 24, 2022
b59e3c0
change
ludeeus Feb 24, 2022
7d19e55
change import
ludeeus Feb 24, 2022
5526a27
Merge branch 'dev' of github.com:home-assistant/core into backup_inte…
ludeeus Feb 28, 2022
09a217f
Update homeassistant/components/backup/manager.py
ludeeus Feb 28, 2022
7965936
coverage
ludeeus Feb 28, 2022
062f00f
adjust
ludeeus Feb 28, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CODEOWNERS
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,8 @@ tests/components/azure_devops/* @timmo001
homeassistant/components/azure_event_hub/* @eavanvalkenburg
tests/components/azure_event_hub/* @eavanvalkenburg
homeassistant/components/azure_service_bus/* @hfurubotten
homeassistant/components/backup/* @home-assistant/core
tests/components/backup/* @home-assistant/core
homeassistant/components/balboa/* @garbled1
tests/components/balboa/* @garbled1
homeassistant/components/beewi_smartclim/* @alemuro
Expand Down
26 changes: 26 additions & 0 deletions homeassistant/components/backup/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
"""The Backup integration."""
from homeassistant.components.hassio import is_hassio
from homeassistant.core import HomeAssistant
from homeassistant.helpers.typing import ConfigType

from .const import DOMAIN, LOGGER
from .http import async_register_http_views
from .manager import BackupManager
from .websocket import async_register_websocket_handlers


async def async_setup(hass: HomeAssistant, config: ConfigType) -> bool:
"""Set up the Backup integration."""
if is_hassio(hass):
LOGGER.warning(
Comment thread
ludeeus marked this conversation as resolved.
Outdated
"The backup integration is not supported on this installation method, "
"please remove it from your configuration"
)
return False

hass.data[DOMAIN] = BackupManager(hass)

async_register_websocket_handlers(hass)
async_register_http_views(hass)

return True
20 changes: 20 additions & 0 deletions homeassistant/components/backup/const.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
"""Constants for the Backup integration."""
from logging import getLogger

from awesomeversion import AwesomeVersion

from homeassistant.const import __version__ as HAVERSION

DOMAIN = "backup"
VERSION = AwesomeVersion(HAVERSION)
LOGGER = getLogger(__package__)

EXCLUDE_FROM_BACKUP = [
Comment thread
ludeeus marked this conversation as resolved.
Comment thread
ludeeus marked this conversation as resolved.
"__pycache__/*",
".DS_Store",
"*.db-shm",
"*.log.*",
"*.log",
"backups/*.tar",
"OZW_Log.txt",
]
48 changes: 48 additions & 0 deletions homeassistant/components/backup/http.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
"""Http view for the Backup integration."""
from __future__ import annotations

from http import HTTPStatus

from aiohttp.hdrs import CONTENT_DISPOSITION
from aiohttp.web import FileResponse, Request, Response

from homeassistant.components.http.view import HomeAssistantView
from homeassistant.core import HomeAssistant, callback
from homeassistant.util import slugify

from .const import DOMAIN
from .manager import BackupManager


@callback
def async_register_http_views(hass: HomeAssistant) -> None:
"""Register the http views."""
hass.http.register_view(DownloadBackupView)


class DownloadBackupView(HomeAssistantView):
"""Generate backup view."""

url = "/api/backup/download/{slug}"
name = "api:backup:download"

async def get( # pylint: disable=no-self-use
self,
request: Request,
slug: str,
) -> FileResponse | Response:
"""Download a backup file."""
if not request["hass_user"].is_admin:
return Response(status=HTTPStatus.UNAUTHORIZED)

manager: BackupManager = request.app["hass"].data[DOMAIN]

if (backup := manager.get_backup(slug)) is None or not backup.path.exists():
return Response(status=HTTPStatus.NOT_FOUND)

return FileResponse(
path=backup.path.as_posix(),
headers={
CONTENT_DISPOSITION: f"attachment; filename={slugify(backup.name)}.tar"
},
)
178 changes: 178 additions & 0 deletions homeassistant/components/backup/manager.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,178 @@
"""Backup manager for the Backup integration."""
from __future__ import annotations

from dataclasses import dataclass
import hashlib
import json
from pathlib import Path
import tarfile
from tempfile import TemporaryDirectory

from homeassistant.core import HomeAssistant
from homeassistant.exceptions import HomeAssistantError
from homeassistant.util import json as json_util
from homeassistant.util.dt import now

from .const import EXCLUDE_FROM_BACKUP, LOGGER, VERSION


@dataclass
class Backup:
"""Backup class."""

slug: str
name: str
date: str
path: Path
size: float

def as_dict(self) -> dict:
"""Return a dict representation of this backup."""
return {
"slug": self.slug,
"name": self.name,
"date": self.date,
"size": self.size,
"path": self.path.as_posix(),
}


class BackupManager:
"""Backup manager for the Backup integration."""

_backups: dict[str, Backup] | None = None

def __init__(self, hass: HomeAssistant) -> None:
"""Initialize the backup manager."""
self.hass = hass
self.backup_dir = Path(hass.config.path("backups"))
self.backing_up = False

@property
def backups(self) -> dict[str, Backup] | None:
"""Return a dict of backups."""
return self._backups

def load_backups(self) -> None:
"""Load data of stored backup files."""
backups = {}

def _read_backup_info(backup_path: Path) -> None:
Comment thread
ludeeus marked this conversation as resolved.
Outdated
with tarfile.open(backup_path, "r:") as backup_file:

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You should use stream like the old tape bands. This flood the memory because it keeps everything into memory just to read a file out of it and the same for creating the backup itself

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

uhh, that is a bug, securetar handle it all as stream which is right

@pvizeli pvizeli Feb 14, 2022

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Update: Yeah, we do that for the top tarfile and use the stream for all others because of the random access to files. So it's not nice but work. So for backup the folder, we should use streams, so never load a full file into memory, just stream over all bites

if data_file := backup_file.extractfile("./backup.json"):
data = json.loads(data_file.read())
backup = Backup(
slug=data["slug"],
name=data["name"],
date=data["date"],
path=backup_path,
size=round(backup_path.stat().st_size / 1_048_576, 2),
)
backups[backup.slug] = backup

for backup_path in self.backup_dir.glob("*.tar"):
_read_backup_info(backup_path)

LOGGER.debug("Loaded %s backups", len(backups))
self._backups = backups

def get_backup(self, slug: str) -> Backup | None:
"""Return a backup."""
return None if self._backups is None else self._backups.get(slug)

def remove_backup(self, slug: str) -> None:
"""Remove a backup."""
LOGGER.error(self._backups)
if self._backups is None or (backup := self.get_backup(slug)) is None:
Comment thread
ludeeus marked this conversation as resolved.
Outdated
return
backup.path.unlink(missing_ok=True)
LOGGER.debug("Removed backup located at %s", backup.path)
self._backups.pop(slug)

async def generate_backup(self) -> Backup:
"""Generate a backup."""
if self.backing_up:
raise HomeAssistantError("Backup already in progress")

if self._backups is None:
self._backups = {}
Comment thread
ludeeus marked this conversation as resolved.
Outdated

try:
self.backing_up = True
backup_name = f"Core {VERSION}"
date_str = now().isoformat()
slug = _generate_slug(date_str, backup_name)

backup_data = {
"slug": slug,
"name": backup_name,
"date": date_str,
"type": "partial",
"folders": ["homeassistant"],
"homeassistant": {} if VERSION.dev else {"version": VERSION},
"compressed": True,
}
tar_file = Path(self.backup_dir, f"{slug}.tar")

if not self.backup_dir.exists():
LOGGER.debug("Creating backup directory")
self.hass.async_add_executor_job(self.backup_dir.mkdir)

def _create_backup() -> None:
Comment thread
ludeeus marked this conversation as resolved.
with TemporaryDirectory() as tmp_dir:
json_util.save_json(
Path(tmp_dir, "backup.json").as_posix(),
backup_data,
)

with tarfile.open(
Path(tmp_dir, "homeassistant.tar.gz"), "w:gz"
) as tar:
_add_directory_to_tarfile(
tar_file=tar,
origin_path=Path(self.hass.config.path()),
)

with tarfile.open(tar_file, "w:") as tar:
tar.add(tmp_dir, arcname=".")

await self.hass.async_add_executor_job(_create_backup)
backup = Backup(
slug=slug,
name=backup_name,
date=date_str,
path=tar_file,
size=round(tar_file.stat().st_size / 1_048_576, 2),
)
self._backups[slug] = backup
Comment thread
ludeeus marked this conversation as resolved.
Outdated
LOGGER.debug("Generated new backup with slug %s", slug)
return backup
finally:
self.backing_up = False


def _add_directory_to_tarfile(
tar_file: tarfile.TarFile,
origin_path: Path,
arcname: str = ".",
) -> None:
"""Add a directory to a tarfile."""
for directory_item in origin_path.iterdir():
arcpath = Path(arcname, directory_item.name).as_posix()

if directory_item.is_file():
if any(directory_item.match(exclude) for exclude in EXCLUDE_FROM_BACKUP):
continue
tar_file.add(directory_item.as_posix(), arcname=arcpath, recursive=False)
continue

if directory_item.is_dir():
_add_directory_to_tarfile(tar_file, directory_item, arcpath)
continue

tar_file.add(directory_item.as_posix(), arcname=arcpath, recursive=False)
Comment thread
ludeeus marked this conversation as resolved.
Outdated


def _generate_slug(date: str, name: str) -> str:
"""Generate a backup slug."""
return hashlib.sha1(f"{date} - {name}".lower().encode()).hexdigest()[:8]
14 changes: 14 additions & 0 deletions homeassistant/components/backup/manifest.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
{
"domain": "backup",
"name": "Backup",
"documentation": "https://www.home-assistant.io/integrations/backup",
"dependencies": [
"http",
"websocket_api"
],
"codeowners": [
"@home-assistant/core"
],
"iot_class": "calculated",
"quality_scale": "internal"
}
74 changes: 74 additions & 0 deletions homeassistant/components/backup/websocket.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
"""Websocket commands for the Backup integration."""
import voluptuous as vol

from homeassistant.components import websocket_api
from homeassistant.core import HomeAssistant, callback

from .const import DOMAIN
from .manager import BackupManager


@callback
def async_register_websocket_handlers(hass: HomeAssistant) -> None:
"""Register websocket commands."""
websocket_api.async_register_command(hass, handle_info)
websocket_api.async_register_command(hass, handle_create)
websocket_api.async_register_command(hass, handle_remove)


@websocket_api.require_admin
@websocket_api.websocket_command({vol.Required("type"): "backup/info"})
@websocket_api.async_response
async def handle_info(
hass: HomeAssistant,
connection: websocket_api.ActiveConnection,
msg: dict,
):
Comment thread
ludeeus marked this conversation as resolved.
"""List all stored backups."""
manager: BackupManager = hass.data[DOMAIN]

if manager.backups is None:
await hass.async_add_executor_job(manager.load_backups)
Comment thread
ludeeus marked this conversation as resolved.
Outdated

connection.send_result(
msg["id"],
{
"backups": list(manager.backups or []),
Comment thread
ludeeus marked this conversation as resolved.
Outdated
"backing_up": manager.backing_up,
},
)


@websocket_api.require_admin
@websocket_api.websocket_command(
{
vol.Required("type"): "backup/remove",
vol.Required("slug"): str,
}
)
@websocket_api.async_response
async def handle_remove(
hass: HomeAssistant,
connection: websocket_api.ActiveConnection,
msg: dict,
):
"""Remove a backup."""
manager: BackupManager = hass.data[DOMAIN]

await hass.async_add_executor_job(manager.remove_backup, msg["slug"])

connection.send_result(msg["id"])


@websocket_api.require_admin
@websocket_api.websocket_command({vol.Required("type"): "backup/generate"})
@websocket_api.async_response
async def handle_create(
hass: HomeAssistant,
connection: websocket_api.ActiveConnection,
msg: dict,
):
"""Generate a backup."""
manager: BackupManager = hass.data[DOMAIN]
backup = await manager.generate_backup()
connection.send_result(msg["id"], backup)
1 change: 1 addition & 0 deletions tests/components/backup/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
"""Tests for the Backup integration."""
Loading