feat: keep yaml comment as is

.github/workflows/update-sigmarule.yaml

@@ -32,22 +32,19 @@ jobs:
           python-version: '3.10'
           token: '${{ secrets.GITHUB_TOKEN }}'
 
-      - name: download requirements.txt with curl 
-        uses: wei/curl@master
-        with:
-          args: https://raw.githubusercontent.com/Yamato-Security/hayabusa-rules/main/tools/sigmac/requirements.txt > requirements.txt 
-
-      - name: setup Python for use script
+      - name: setup Poetry
         run: |
-          pip install -r requirements.txt
-          rm requirements.txt
+          curl -sSL https://install.python-poetry.org | python3 -
 
       - name: Update sigma rules
         run: |
-          python3 hayabusa-rules/tools/sigmac/logsource_mapping.py -r sigma-repo -o converted_rules
+          cd hayabusa-rules/tools/sigmac/
+          poetry install --no-root
+          poetry run python logsource_mapping.py -r ../../../sigma-repo -o converted_rules
+          cd -
           rm -rf hayabusa-rules/sigma/
           mkdir hayabusa-rules/sigma/
-          cp -r converted_rules/* hayabusa-rules/sigma/
+          cp -r hayabusa-rules/tools/sigmac/converted_rules/* hayabusa-rules/sigma/
 
       - name: Create Text
         id: create-text

CHANGELOG-Japanese.md

@@ -2,6 +2,7 @@
 
 ## v2.13.0-dev [2024/01/19]
 
+Sigmaルールのコメントを残すようにした。以前は変換後に削除されていた。(#568) (@fukusuket)
 Sigma変換バックエンドのパッケージ管理は [Poetry](https://python-poetry.org/) 、静的コード分析は [Ruff](https://github.com/astral-sh/ruff) で実行するようにした。(#567) (@fukusuket)
 
 ## v2.12.0 [2023/12/19]

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## v2.13.0-dev [2024/01/19]
 
+Comments in Sigma rules are left as is. Before, they would be stripped after conversion. (#568) (@fukusuket)
 Package management for the sigma conversion backend is now handled by [Poetry](https://python-poetry.org/) and static code analysis is performed by [Ruff](https://github.com/astral-sh/ruff). (#567) (@fukusuket)
 
 ## v2.12.0 [2023/12/19]

tools/sigmac/logsource_mapping.py

@@ -14,7 +14,7 @@
 from pathlib import Path
 from typing import Union, Optional
 
-import oyaml as yaml
+import ruamel.yaml
 
 FORMAT = '[%(levelname)-2s:%(filename)s:%(lineno)d] %(message)s'
 logging.basicConfig(format=FORMAT, level=logging.INFO)
@@ -181,17 +181,6 @@ def is_detectable(self, obj: dict) -> bool:
         return True
 
 
-class IndentDumper(yaml.Dumper):
-    """
-    pyyamlの↓バグで、valueがlistの場合インデントされないため、yaml.dump時にインデントさせるためのカスタムクラス
-    https://github.com/yaml/pyyaml/issues/234
-    https://stackoverflow.com/questions/25108581/python-yaml-dump-bad-indentation/39681672#39681672
-    """
-
-    def increase_indent(self, flow=False, indentless=False):
-        return super(IndentDumper, self).increase_indent(flow, False)
-
-
 @dataclass(frozen=True)
 class LogsourceConverter:
     sigma_path: str
@@ -325,7 +314,10 @@ def dump_yml(self, base_dir: str, out_dir: str) -> list[tuple[str, str]]:
         for is_sysmon, obj in self.sigma_converted:
             output_path = build_out_path(base_dir, out_dir, self.sigma_path, is_sysmon)
             with StringIO() as bs:
-                yaml.dump(obj, bs, Dumper=IndentDumper, default_flow_style=False, indent=4)
+                yaml = ruamel.yaml.YAML()
+                yaml.width = 4096
+                yaml.indent(mapping=4, sequence=4, offset=4)
+                yaml.dump(obj, bs)
                 res.append((output_path, bs.getvalue()))
         return res
 
@@ -365,7 +357,8 @@ def create_obj(base_dir: Optional[str], file_name: str) -> dict:
         sys.exit(1)
     try:
         with open(file_path, encoding="utf-8") as f:
-            d = yaml.safe_load(f)
+            yaml = ruamel.yaml.YAML()
+            d = yaml.load(f)
             LOGGER.debug(f"loading yaml [{file_path}] done successfully.")
             return d
     except Exception as e:
@@ -447,7 +440,8 @@ def find_windows_sigma_rule_files(root: str, rule_pattern: str):
                 continue  # フォルダパスにrule/deprecated/unsupportedがつかないものは、Sigmaルールと関係ないため、除外
             try:
                 with open(filepath, encoding="utf-8") as f:
-                    data = yaml.safe_load(f)
+                    yaml = ruamel.yaml.YAML()
+                    data = yaml.load(f)
                 if data.get('logsource', {}).get('category') != "antivirus" \
                         and data.get('logsource', {}).get('product') != 'windows':
                     LOGGER.debug(f"[{filepath}] has no windows rule. Conversion skipped.")
@@ -511,6 +505,12 @@ def find_windows_sigma_rule_files(root: str, rule_pattern: str):
                 p = Path(out_path)
                 if not p.parent.exists():
                     os.makedirs(p.parent)
+                # ruamelは以下のインデントを正しく処理できないので、文字列置換で対応する
+                parsed_yaml = parsed_yaml.replace("    type:", "      type:")
+                parsed_yaml = parsed_yaml.replace("      EventType:", "        EventType:")
+                parsed_yaml = parsed_yaml.replace("      OperationType:", "        OperationType:")
+                parsed_yaml = parsed_yaml.replace("      NewProcessName", "        NewProcessName")
+                parsed_yaml = parsed_yaml.replace("      CommandLine", "        CommandLine")
                 p.write_text(parsed_yaml, encoding="utf-8")  # 変換後のSigmaルール(yml形式の文字列)をファイルに出力
                 file_cnt += 1
                 LOGGER.debug(f"Converted to [{out_path}] done.")