Merge pull request #11 from azuki774/add-mf

Add money-forward fetcher

.github/workflows/test.yml

@@ -0,0 +1,21 @@
+name: Pytest
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - "**"
+
+jobs:
+  pytest:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.13'
+    - name: install requirements.txt
+      run: pip install -r src/moneyforward/requirements.txt
+    - name: Run pytest
+      run: pytest -v

.gitignore

@@ -30,3 +30,4 @@ deployment/*.jpg
 compose.yml
 *-token.env
 tmp
+src/moneyforward/__pycache__/

build/moneyforward/Dockerfile

@@ -0,0 +1,39 @@
+FROM python:3.13-bookworm
+
+# Required Packages
+RUN apt-get update && \
+    apt-get install -y \
+    curl \
+    unzip \
+    wget \
+    unzip \
+    libglib2.0-0 \
+    libnss3 \
+    libgconf-2-4 \
+    libfontconfig1 && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install Chrome
+RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add -
+RUN echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list
+RUN apt-get update && apt-get install -y google-chrome-stable && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# Install driver (Ref. https://sleepless-se.net/2024/03/19/python-selenium-docker/)
+RUN wget -O chrome.json https://googlechromelabs.github.io/chrome-for-testing/known-good-versions-with-downloads.json && \
+    LINUX_STABLE_URL=$(grep -oP '"url":".*?(?=")' chrome.json | grep 'linux64' | head -n 1 | cut -d'"' -f4) && \
+    wget -O chrome.zip $LINUX_STABLE_URL && \
+    unzip chrome.zip && \
+    rm chrome.zip chrome.json
+
+# AWS Setup
+RUN curl -o /var/tmp/awscli.zip https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip && \
+    unzip -d /usr/local/bin/ /var/tmp/awscli.zip
+
+# Install Python module
+COPY /src/moneyforward/requirements.txt /tmp/
+RUN pip install --upgrade pip && pip install -r /tmp/requirements.txt && mkdir -p /data
+COPY --chmod=755 build/moneyforward/main.sh /src/main.sh
+COPY src/moneyforward/ /src/
+
+ENTRYPOINT ["/src/main.sh"]

build/moneyforward/main.sh

@@ -11,23 +11,15 @@ YYYYMMDD=`date '+%Y%m%d'`
 # AWS_SECRET_ACCESS_KEY # from env
 # user="xxxxxxxxx" # moneyforward id  , from env
 # pass="yyyyyyyyy" # moneyforward pass, from env
-# wsAddr # from env (ex: localhost:7327)
 
-SCRAPERS_BIN="/usr/local/bin/myscrapers"
 AWS_BIN="/usr/local/bin/aws/dist/aws"
-outputDir="/data/${YYYYMM}/${YYYYMMDD}"
-
-REMOTE_DIR="${BUCKET_DIR}/${YYYYMM}/${YYYYMMDD}"
-
-function download () {
-    echo "job start"
-    mkdir -p ${outputDir}
-    echo "output to dir: ${outputDir}"
-    outputDir=${outputDir} \
-    user=${user} \
-    pass=${pass} \
-    ${SCRAPERS_BIN} download moneyforward --lastmonth
-    echo "job complete"
+DATA_DIR="/data"
+REMOTE_DIR="${BUCKET_DIR}"
+
+function fetch () {
+    echo "fetcher start"
+    python3 -u /src/main.py
+    echo "fetcher complete"
 }
 
 function create_s3_credentials () {
@@ -49,13 +41,15 @@ function create_s3_credentials () {
 
 function s3_upload () {
     echo "s3 upload start"
-    ${AWS_BIN} s3 cp ${outputDir}/ "s3://${BUCKET_NAME}/${REMOTE_DIR}" --recursive --endpoint-url="${BUCKET_URL}"
+    ${AWS_BIN} s3 cp ${DATA_DIR}/ "s3://${BUCKET_NAME}/${REMOTE_DIR}/" --recursive --endpoint-url="${BUCKET_URL}"
     echo "s3 upload complete"
 }
 
-download
+fetch
 
-if [ -n $BUCKET_NAME ]; then
-    create_s3_credentials
-    # s3_upload
+if [ -z $BUCKET_NAME ]; then
+    exit 0
 fi
+
+create_s3_credentials
+s3_upload

deployment/compose.yml

@@ -1,19 +1,8 @@
 services:
-  myscrapers-sbi-test:
-    image: myscrapers-sbi:latest
-    container_name: myscrapers-sbi
-    environment:
-      - chromeAddr=http://example.com:4444/wd/hub # your value
-      - TZ="JST-9"
-    env_file:
-      - sbi-token.env
-    volumes:
-      - ./browser/:/data/
   myscrapers-mf-test:
     image: myscrapers-mf:latest
     container_name: myscrapers-mf
     environment:
-      - chromeAddr=http://example.com:4444/wd/hub # your value
       - TZ="JST-9"
     env_file:
       - moneyforward-token.env

docs/README.md

@@ -3,11 +3,16 @@
 ### output CSV
 例：
 ```
-計算対象,日付,内容,金額（円）,保有金融機関,大項目,中項目,メモ,振替,削除
-,07/16(火),ローソン,-291,三井住友カード,食費,食料品,,,
-,07/16(火),GITHUB,-158,JCBカード,通信費,情報サービス,,,
-,07/10(水),マクドナルド,-600,三井住友カード,食費,外食,,,
+"計算対象","日付","内容","金額（円）","保有金融機関","大項目","中項目","メモ","振替","ID"
+"1","2024/12/12","マクドナルド","-600","三井住友カード","食費","外食","","",""
+"1","2024/12/12","JR東日本モバイルSuica●","-3,000","三井住友カード","","","","","
 ```
+- moneyforward 公式との差異は下記
+    - 計算対象は無条件で1にする
+    - ID 部分は取得不可なので、空文字にする
+    - 振替欄も正しく入らない（空文字）
+    - ただし、文字コードは UTF8 のままにする（公式はSJIS）
+
 
 ### 出力先
 - コンテナ内デフォルト: `/data/YYYYMM/YYYYMMDD/cf.csv`, `--lastmonth` 付与時は、`/data/YYYYMM/YYYYMMDD/cf_lastmonth.csv` も出力。

src/moneyforward/driver.py

@@ -0,0 +1,18 @@
+from selenium import webdriver
+from selenium.webdriver.chrome.service import Service as ChromeService
+from webdriver_manager.chrome import ChromeDriverManager
+import os
+
+def get_driver():
+    options=webdriver.ChromeOptions()
+    options.add_argument("--headless")
+    options.add_argument("--no-sandbox")
+    options.add_argument("--disable-gpu")
+    options.add_argument("--lang=ja-JP")
+    options.add_argument("--disable-dev-shm-usage")
+    # options.add_experimental_option("prefs", {"download.default_directory": "/data/" })
+    UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36"
+    options.add_argument("--user-agent=" + UA)
+    driver = webdriver.Chrome(options=options)
+    driver.implicitly_wait(10)
+    return driver