Skip to content

Commit 46904cf

Browse files
KazuhiroMKazuhiroM
committed
Merge branch 'master' of github.com:The-Japan-DataScientist-Society/100knocks-preprocess
2 parents 0524f99 + 155e26f commit 46904cf

File tree

5 files changed

+346
-1
lines changed

5 files changed

+346
-1
lines changed

‎.github/workflows/run_notebooks.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,6 @@ jobs:
2222
echo "Run docker/work/${f}"
2323
2424
# 「UsageError: %%sql is a cell magic, but the cell body is empty. Did you mean the line magic %sql (single %)? 」を許容するため、UsageErrorを許容する設定にしている
25-
docker-compose exec -T notebook bash -c "cd work && jupyter nbconvert --to notebook --debug --NotebookClient.allow_error_names UsageError --execute ${f}"
25+
docker compose exec -T notebook bash -c "cd work && jupyter nbconvert --to notebook --debug --NotebookClient.allow_error_names UsageError --execute ${f}"
2626
echo
2727
done

‎.github/workflows/update_pre_commit_config.yml‎

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
name: update-pre-commit-config
2+
3+
on:
4+
pull_request:
5+
6+
jobs:
7+
update-pre-commit-config:
8+
runs-on: ubuntu-latest
9+
steps:
10+
- uses: actions/checkout@v2
11+
with:
12+
fetch-depth: 0
13+
ref: ${{ github.event.pull_request.head.sha }}
14+
- name: Install packages
15+
run: npm install js-yaml
16+
- name: Update .pre-commit-config.yaml
17+
uses: actions/github-script@v5
18+
with:
19+
github-token: ${{secrets.GITHUB_TOKEN}}
20+
script: |
21+
const fs = require('fs')
22+
const yaml = require('js-yaml')
23+
24+
const config_filename = '.pre-commit-config.yaml'
25+
const config = yaml.load(fs.readFileSync(config_filename, 'utf8'))
26+
const common_params = {
27+
owner: 'zricethezav',
28+
repo: 'gitleaks'
29+
}
30+
console.log("call repos.getLatestRelease:", common_params)
31+
const latest_release = (await github.rest.repos.getLatestRelease(common_params)).data
32+
config.repos = config.repos.map(repo => {
33+
if (repo.repo === 'https://github.com/' + common_params.owner + '/' + common_params.repo) {
34+
repo.rev = latest_release.tag_name;
35+
}
36+
37+
return repo;
38+
})
39+
40+
try {
41+
fs.writeFileSync(config_filename, yaml.dump(config), 'utf8')
42+
} catch (err) {
43+
console.error(err.message)
44+
process.exit(1)
45+
}
46+
# 差分があったときは差分を出力する
47+
- name: Show diff
48+
id: diff
49+
run: |
50+
result=$(git diff .pre-commit-config.yaml)
51+
echo "::set-output name=result::$result"
52+
# 差分があったときは、コミットを作りpushする
53+
- name: Push
54+
env:
55+
HEAD_REF: ${{github.event.pull_request.head.ref}}
56+
if: ${{ github.event.pull_request.head.repo.full_name == github.repository && steps.diff.outputs.result != '' }}
57+
run: |
58+
git config user.name "100knocks preprocess CI"
59+
git config user.email "100knocks-preprocess-ci@example.com"
60+
git add .pre-commit-config.yaml
61+
git commit -m "Update .pre-commit-config.yaml"
62+
git push -f https://${{github.actor}}:${{secrets.GITHUB_TOKEN}}@github.com/${{github.repository}}.git "HEAD:refs/heads/fix-version-pre-commit-config-${HEAD_REF}"
63+
- name: Get PullRequests
64+
uses: actions/github-script@v5
65+
env:
66+
HEAD_REF: ${{github.event.pull_request.head.ref}}
67+
if: ${{ github.event.pull_request.head.repo.full_name == github.repository && steps.diff.outputs.result != '' }}
68+
id: get_pull_requests
69+
with:
70+
github-token: ${{secrets.GITHUB_TOKEN}}
71+
script: |
72+
const HEAD_REF = process.env["HEAD_REF"]
73+
const pulls_list_params = {
74+
owner: context.repo.owner,
75+
repo: context.repo.repo,
76+
head: "The-Japan-DataScientist-Society:fix-version-pre-commit-config-" + HEAD_REF,
77+
base: HEAD_REF,
78+
state: "open"
79+
}
80+
console.log("call pulls.list:", pulls_list_params)
81+
const pulls = await github.paginate(github.rest.pulls.list, pulls_list_params)
82+
return pulls.length
83+
# pushしたブランチでPRを作る
84+
- name: Create PullRequest
85+
uses: actions/github-script@v5
86+
env:
87+
HEAD_REF: ${{github.event.pull_request.head.ref}}
88+
if: ${{ github.event.pull_request.head.repo.full_name == github.repository && steps.diff.outputs.result != '' && steps.get_pull_requests.outputs.result == 0 }}
89+
id: create_pull_request
90+
with:
91+
github-token: ${{secrets.GITHUB_TOKEN}}
92+
script: |
93+
const HEAD_REF = process.env["HEAD_REF"]
94+
const pulls_create_params = {
95+
owner: context.repo.owner,
96+
repo: context.repo.repo,
97+
head: "The-Japan-DataScientist-Society:fix-version-pre-commit-config-" + HEAD_REF,
98+
base: HEAD_REF,
99+
title: ".pre-commit-config.yamlアップデート #${{github.event.pull_request.number}}",
100+
body: ".pre-commit-config.yamlをアップデートしました。マージすると元のPRにアップデートが反映されます。 #${{github.event.pull_request.number}}"
101+
}
102+
console.log("call pulls.create:", pulls_create_params)
103+
const create_pull_res = (await github.rest.pulls.create(pulls_create_params)).data
104+
return create_pull_res.number
105+
- name: Assign a user
106+
uses: actions/github-script@v5
107+
if: ${{ github.event.pull_request.head.repo.full_name == github.repository && steps.diff.outputs.result != '' && steps.get_pull_requests.outputs.result == 0 && github.event.pull_request.user.login != 'dependabot[bot]' }}
108+
with:
109+
github-token: ${{secrets.GITHUB_TOKEN}}
110+
script: |
111+
const issues_add_assignees_params = {
112+
owner: context.repo.owner,
113+
repo: context.repo.repo,
114+
issue_number: ${{steps.create_pull_request.outputs.result}},
115+
assignees: ["${{github.event.pull_request.user.login}}"]
116+
}
117+
console.log("call issues.addAssignees:", issues_add_assignees_params)
118+
await github.rest.issues.addAssignees(issues_add_assignees_params)
119+
# 既にアップデートのPRがある状態で、手動でアップデートした場合、アップデートのPRを閉じる
120+
- name: Close PullRequest
121+
uses: actions/github-script@v5
122+
env:
123+
HEAD_REF: ${{github.event.pull_request.head.ref}}
124+
if: ${{ github.event.pull_request.head.repo.full_name == github.repository && steps.diff.outputs.result == '' }}
125+
with:
126+
github-token: ${{secrets.GITHUB_TOKEN}}
127+
script: |
128+
const HEAD_REF = process.env["HEAD_REF"]
129+
const head_name = "fix-version-pre-commit-config-" + HEAD_REF
130+
const common_params = {
131+
owner: context.repo.owner,
132+
repo: context.repo.repo
133+
}
134+
const pulls_list_params = {
135+
head: "The-Japan-DataScientist-Society:" + head_name,
136+
base: HEAD_REF,
137+
state: "open",
138+
...common_params
139+
}
140+
console.log("call pulls.list:", pulls_list_params)
141+
const pulls = await github.paginate(github.rest.pulls.list, pulls_list_params)
142+
143+
for (const pull of pulls) {
144+
const pulls_update_params = {
145+
pull_number: pull.number,
146+
state: "closed",
147+
...common_params
148+
}
149+
console.log("call pulls.update:", pulls_update_params)
150+
await github.rest.pulls.update(pulls_update_params)
151+
const git_deleteRef_params = {
152+
ref: "heads/" + head_name,
153+
...common_params
154+
}
155+
console.log("call git.deleteRef:", git_deleteRef_params)
156+
await github.rest.git.deleteRef(git_deleteRef_params)
157+
}
158+
- name: Exit
159+
if: ${{ steps.diff.outputs.result != '' }}
160+
run: exit 1

‎.gitleaks.toml‎

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
title = "gitleaks config"
2+
3+
[[rules]]
4+
description = "AWS Access Key"
5+
regex = '''(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}'''
6+
tags = ["key", "AWS"]
7+
8+
[[rules]]
9+
description = "AWS Secret Key"
10+
regex = '''(?i)aws(.{0,20})?(?-i)['\"][0-9a-zA-Z\/+]{40}['\"]'''
11+
tags = ["key", "AWS"]
12+
13+
[[rules]]
14+
description = "AWS MWS key"
15+
regex = '''amzn\.mws\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'''
16+
tags = ["key", "AWS", "MWS"]
17+
18+
[[rules]]
19+
description = "Facebook Secret Key"
20+
regex = '''(?i)(facebook|fb)(.{0,20})?(?-i)['\"][0-9a-f]{32}['\"]'''
21+
tags = ["key", "Facebook"]
22+
23+
[[rules]]
24+
description = "Facebook Client ID"
25+
regex = '''(?i)(facebook|fb)(.{0,20})?['\"][0-9]{13,17}['\"]'''
26+
tags = ["key", "Facebook"]
27+
28+
[[rules]]
29+
description = "Twitter Secret Key"
30+
regex = '''(?i)twitter(.{0,20})?['\"][0-9a-z]{35,44}['\"]'''
31+
tags = ["key", "Twitter"]
32+
33+
[[rules]]
34+
description = "Twitter Client ID"
35+
regex = '''(?i)twitter(.{0,20})?['\"][0-9a-z]{18,25}['\"]'''
36+
tags = ["client", "Twitter"]
37+
38+
[[rules]]
39+
description = "Github Personal Access Token"
40+
regex = '''ghp_[0-9a-zA-Z]{36}'''
41+
tags = ["key", "Github"]
42+
[[rules]]
43+
description = "Github OAuth Access Token"
44+
regex = '''gho_[0-9a-zA-Z]{36}'''
45+
tags = ["key", "Github"]
46+
[[rules]]
47+
description = "Github App Token"
48+
regex = '''(ghu|ghs)_[0-9a-zA-Z]{36}'''
49+
tags = ["key", "Github"]
50+
[[rules]]
51+
description = "Github Refresh Token"
52+
regex = '''ghr_[0-9a-zA-Z]{76}'''
53+
tags = ["key", "Github"]
54+
55+
[[rules]]
56+
description = "LinkedIn Client ID"
57+
regex = '''(?i)linkedin(.{0,20})?(?-i)[0-9a-z]{12}'''
58+
tags = ["client", "LinkedIn"]
59+
60+
[[rules]]
61+
description = "LinkedIn Secret Key"
62+
regex = '''(?i)linkedin(.{0,20})?[0-9a-z]{16}'''
63+
tags = ["secret", "LinkedIn"]
64+
65+
[[rules]]
66+
description = "Slack"
67+
regex = '''xox[baprs]-([0-9a-zA-Z]{10,48})?'''
68+
tags = ["key", "Slack"]
69+
70+
[[rules]]
71+
description = "Asymmetric Private Key"
72+
regex = '''-----BEGIN ((EC|PGP|DSA|RSA|OPENSSH) )?PRIVATE KEY( BLOCK)?-----'''
73+
tags = ["key", "AsymmetricPrivateKey"]
74+
75+
[[rules]]
76+
description = "Google API key"
77+
regex = '''AIza[0-9A-Za-z\\-_]{35}'''
78+
tags = ["key", "Google"]
79+
80+
[[rules]]
81+
description = "Google (GCP) Service Account"
82+
regex = '''"type": "service_account"'''
83+
tags = ["key", "Google"]
84+
85+
[[rules]]
86+
description = "Heroku API key"
87+
regex = '''(?i)heroku(.{0,20})?[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'''
88+
tags = ["key", "Heroku"]
89+
90+
[[rules]]
91+
description = "MailChimp API key"
92+
regex = '''(?i)(mailchimp|mc)(.{0,20})?[0-9a-f]{32}-us[0-9]{1,2}'''
93+
tags = ["key", "Mailchimp"]
94+
95+
[[rules]]
96+
description = "Mailgun API key"
97+
regex = '''((?i)(mailgun|mg)(.{0,20})?)?key-[0-9a-z]{32}'''
98+
tags = ["key", "Mailgun"]
99+
100+
[[rules]]
101+
description = "PayPal Braintree access token"
102+
regex = '''access_token\$production\$[0-9a-z]{16}\$[0-9a-f]{32}'''
103+
tags = ["key", "Paypal"]
104+
105+
[[rules]]
106+
description = "Picatic API key"
107+
regex = '''sk_live_[0-9a-z]{32}'''
108+
tags = ["key", "Picatic"]
109+
110+
[[rules]]
111+
description = "SendGrid API Key"
112+
regex = '''SG\.[\w_]{16,32}\.[\w_]{16,64}'''
113+
tags = ["key", "SendGrid"]
114+
115+
[[rules]]
116+
description = "Slack Webhook"
117+
regex = '''https://hooks.slack.com/services/T[a-zA-Z0-9_]{8}/B[a-zA-Z0-9_]{8,12}/[a-zA-Z0-9_]{24}'''
118+
tags = ["key", "slack"]
119+
120+
[[rules]]
121+
description = "Stripe API key"
122+
regex = '''(?i)stripe(.{0,20})?[sr]k_live_[0-9a-zA-Z]{24}'''
123+
tags = ["key", "Stripe"]
124+
125+
[[rules]]
126+
description = "Square access token"
127+
regex = '''sq0atp-[0-9A-Za-z\-_]{22}'''
128+
tags = ["key", "square"]
129+
130+
[[rules]]
131+
description = "Square OAuth secret"
132+
regex = '''sq0csp-[0-9A-Za-z\\-_]{43}'''
133+
tags = ["key", "square"]
134+
135+
[[rules]]
136+
description = "Twilio API key"
137+
regex = '''(?i)twilio(.{0,20})?SK[0-9a-f]{32}'''
138+
tags = ["key", "twilio"]
139+
140+
[[rules]]
141+
description = "Dynatrace ttoken"
142+
regex = '''dt0[a-zA-Z]{1}[0-9]{2}\.[A-Z0-9]{24}\.[A-Z0-9]{64}'''
143+
tags = ["key", "Dynatrace"]
144+
145+
[[rules]]
146+
description = "Shopify shared secret"
147+
regex = '''shpss_[a-fA-F0-9]{32}'''
148+
tags = ["key", "Shopify"]
149+
150+
[[rules]]
151+
description = "Shopify access token"
152+
regex = '''shpat_[a-fA-F0-9]{32}'''
153+
tags = ["key", "Shopify"]
154+
155+
[[rules]]
156+
description = "Shopify custom app access token"
157+
regex = '''shpca_[a-fA-F0-9]{32}'''
158+
tags = ["key", "Shopify"]
159+
160+
[[rules]]
161+
description = "Shopify private app access token"
162+
regex = '''shppa_[a-fA-F0-9]{32}'''
163+
tags = ["key", "Shopify"]
164+
165+
[[rules]]
166+
description = "PyPI upload token"
167+
regex = '''pypi-AgEIcHlwaS5vcmc[A-Za-z0-9-_]{50,1000}'''
168+
tags = ["key", "pypi"]
169+
170+
[allowlist]
171+
description = "Allowlisted files"
172+
files = ['''^\.?gitleaks.toml$''',
173+
'''(.*?)(png|jpg|gif|doc|docx|pdf|bin|xls|pyc|zip)$''',
174+
'''(go.mod|go.sum)$''']

‎.pre-commit-config.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
repos:
2+
- repo: https://github.com/zricethezav/gitleaks
3+
rev: v8.8.11
4+
hooks:
5+
- id: gitleaks

‎README.rst‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,12 @@ Usage
5252
- ブラウザで以下のURLにアクセスします
5353
http://localhost:8888
5454

55+
How to contribute
56+
====
57+
開発に協力していただける場合は本リポジトリをcloneし、 https://pre-commit.com/ の手順に従って `pre-commit` をインストールしてください。
58+
59+
これにより、 `.pre-commit-config.yaml <.pre-commit-config.yaml>`_ の設定に基づいて、コミット時にクレデンシャルが含まれていないかの検査が行われるようになります。
60+
5561
Document
5662
====
5763
- doc配下にデータサイエンス100本ノック（構造化データ加工編）の説明資料と設問PDF、設問HTML、解答例HTMLを配置

0 commit comments

Comments
 (0)