Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

setup: lazy creation of records #635

Merged
merged 1 commit into from
Nov 22, 2019
Merged

setup: lazy creation of records #635

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion rero_ils/modules/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ def read_json_record(json_file, buf_size=1024, decoder=JSONDecoder()):
:param decoder: decoder to use for decoding
:return: record Generator
"""
buffer = json_file.read(buf_size).replace('\n', '')
buffer = json_file.read(2).replace('\n', '')
# we have to delete the first [ for an list of records
if buffer.startswith('['):
buffer = buffer[1:].lstrip()
Expand Down
37 changes: 19 additions & 18 deletions scripts/setup
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,10 @@ DEPLOYMENT=false
CREATE_ITEMS_HOLDINGS_SMALL=false
CREATE_ITEMS_HOLDINGS_BIG=false
STOP_EXECUTION=true
CREATE_LAZY=""

# options may be followed by one colon to indicate they have a required argument
if ! options=$(getopt -o dsb -l deployment,create_items_holdings_small,create_items_holdings_big,data_path: -- "$@")
if ! options=$(getopt -o dsb -l deployment,create_items_holdings_small,create_items_holdings_big,lazy,data_path: -- "$@")
then
# something went wrong, getopt will put out an error message for us
exit 1
Expand All @@ -57,6 +58,7 @@ do
-s|--create_items_holdings_small) CREATE_ITEMS_HOLDINGS_SMALL=true ;;
-b|--create_items_holdings_big) CREATE_ITEMS_HOLDINGS_BIG=true ;;
-c|--continue) STOP_EXECUTION=false ;;
-l|--lazy) CREATE_LAZY="--lazy" ;;
-D|--data_path) DATA_PATH=$2 ;;
(--) shift; break;;
(-*) display_error_message "$0: error - unrecognized option $1"; exit 1;;
Expand Down Expand Up @@ -139,23 +141,23 @@ pipenv run invenio access allow superuser-access role superusers
pipenv run invenio roles add [email protected] admins
pipenv run invenio roles add [email protected] superusers

display_success_message "Organisations:"
pipenv run invenio fixtures create --pid_type org ${DATA_PATH}/organisations.json --append
display_success_message "Organisations: ${CREATE_LAZY}"
pipenv run invenio fixtures create --pid_type org ${DATA_PATH}/organisations.json --append ${CREATE_LAZY}
pipenv run invenio index reindex -t org --yes-i-know
display_success_message "Libraries:"
pipenv run invenio fixtures create --pid_type lib ${DATA_PATH}/libraries.json --append
display_success_message "Libraries: ${CREATE_LAZY}"
pipenv run invenio fixtures create --pid_type lib ${DATA_PATH}/libraries.json --append ${CREATE_LAZY}
pipenv run invenio index reindex -t lib --yes-i-know
display_success_message "Locations:"
pipenv run invenio fixtures create --pid_type loc ${DATA_PATH}/locations.json --append
display_success_message "Locations: ${CREATE_LAZY}"
pipenv run invenio fixtures create --pid_type loc ${DATA_PATH}/locations.json --append ${CREATE_LAZY}
pipenv run invenio index reindex -t loc --yes-i-know
display_success_message "Item types:"
pipenv run invenio fixtures create --pid_type itty ${DATA_PATH}/item_types.json --append
display_success_message "Item types: ${CREATE_LAZY}"
pipenv run invenio fixtures create --pid_type itty ${DATA_PATH}/item_types.json --append ${CREATE_LAZY}
pipenv run invenio index reindex -t itty --yes-i-know
display_success_message "Patron types:"
pipenv run invenio fixtures create --pid_type ptty ${DATA_PATH}/patron_types.json --append
display_success_message "Patron types: ${CREATE_LAZY}"
pipenv run invenio fixtures create --pid_type ptty ${DATA_PATH}/patron_types.json --append ${CREATE_LAZY}
pipenv run invenio index reindex -t ptty --yes-i-know
display_success_message "Circulation policies:"
pipenv run invenio fixtures create --pid_type cipo ${DATA_PATH}/circulation_policies.json --append
display_success_message "Circulation policies: ${CREATE_LAZY}"
pipenv run invenio fixtures create --pid_type cipo ${DATA_PATH}/circulation_policies.json --append ${CREATE_LAZY}
pipenv run invenio index reindex -t cipo --yes-i-know
pipenv run invenio index run --raise-on-error

Expand Down Expand Up @@ -209,23 +211,22 @@ then
fi
fi

display_success_message "Holdings:"
display_success_message "Holdings: ${CREATE_LAZY}"
echo -e ${HOLDINGS}
pipenv run invenio fixtures create --pid_type hold --schema 'http://ils.rero.ch/schema/holdings/holding-v0.0.1.json' ${HOLDINGS} --append
pipenv run invenio fixtures create --pid_type hold --schema 'http://ils.rero.ch/schema/holdings/holding-v0.0.1.json' ${HOLDINGS} --append ${CREATE_LAZY}
pipenv run invenio index reindex -t hold --yes-i-know
pipenv run invenio index run -c 4 --raise-on-error

display_success_message "Items:"
display_success_message "Items: ${CREATE_LAZY}"
echo -e ${ITEMS}
pipenv run invenio fixtures create --pid_type item --schema 'http://ils.rero.ch/schema/items/item-v0.0.1.json' ${ITEMS} --append
pipenv run invenio fixtures create --pid_type item --schema 'http://ils.rero.ch/schema/items/item-v0.0.1.json' ${ITEMS} --append ${CREATE_LAZY}
pipenv run invenio index reindex -t item --yes-i-know
pipenv run invenio index run -c 4 --raise-on-error

display_success_message "Index Documents:"
pipenv run invenio index reindex -t doc --yes-i-know
pipenv run invenio index run -c 4 --raise-on-error

set +e
display_success_message "Circulation transactions:"
pipenv run invenio fixtures create_loans --fee ${DATA_PATH}/loans.json

Expand Down