WARNING: THIS SITE IS A MIRROR OF GITHUB.COM / IT CANNOT LOGIN OR REGISTER ACCOUNTS / THE CONTENTS ARE PROVIDED AS-IS / THIS SITE ASSUMES NO RESPONSIBILITY FOR ANY DISPLAYED CONTENT OR LINKS / IF YOU FOUND SOMETHING MAY NOT GOOD FOR EVERYONE, CONTACT ADMIN AT ilovescratch@foxmail.com
Skip to content

refactor: Replace information_schema.tables for db size with du (backport #4031) #4044

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mergify wants to merge 1 commit into
base: master
Choose a base branch
from
Open

refactor: Replace information_schema.tables for db size with du (backport #4031) #4044

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 27 additions & 10 deletions press/agent.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import re
from contextlib import suppress
from datetime import date
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any

import frappe
import frappe.utils
Expand Down Expand Up @@ -191,7 +191,7 @@ def restore_site(self, site: "Site", skip_failing_patches=False):
)

def rename_site(self, site, new_name: str, create_user: dict | None = None, config: dict | None = None):
data = {"new_name": new_name}
data: dict[str, Any] = {"new_name": new_name}
if create_user:
data["create_user"] = create_user
if config:
Expand Down Expand Up @@ -1017,7 +1017,7 @@ def create_agent_job(
self,
job_type,
path,
data=None,
data: dict | None = None,
files=None,
method="POST",
bench=None,
Expand All @@ -1038,12 +1038,12 @@ def create_agent_job(
)

if not disable_agent_job_deduplication:
job = self.get_similar_in_execution_job(
existing_job = self.get_similar_in_execution_job(
job_type, path, bench, site, code_server, upstream, host, method
)

if job:
return job
if existing_job:
return existing_job

job: "AgentJob" = frappe.get_doc(
{
Expand Down Expand Up @@ -1375,6 +1375,23 @@ def kill_database_process(self, site, id):
},
)

def update_database_schema_sizes(self):
if self.server_type != "Database Server":
return NotImplementedError("This method is only supported for Database Server")

return self.create_agent_job(
"Update Database Schema Sizes",
"database/update-schema-sizes",
data={
"private_ip": frappe.get_value("Database Server", self.server, "private_ip"),
"mariadb_root_password": get_decrypted_password(
"Database Server", self.server, "mariadb_root_password"
),
},
reference_doctype=self.server_type,
reference_name=self.server,
)

def fetch_database_variables(self):
if self.server_type != "Database Server":
return NotImplementedError("Only Database Server supports this method")
Expand Down Expand Up @@ -1591,14 +1608,14 @@ def backup_site_database_from_snapshot(
):
from press.press.doctype.site_backup.site_backup import get_backup_bucket

database_server: DatabaseServer = frappe.get_doc("Database Server", database_server)
database_server_doc: DatabaseServer = frappe.get_doc("Database Server", database_server) # type: ignore
data = {
"site": site,
"database_name": database_name,
"database_ip": frappe.get_value(
"Virtual Machine", database_server.virtual_machine, "private_ip_address"
"Virtual Machine", database_server_doc.virtual_machine, "private_ip_address"
),
"mariadb_root_password": database_server.get_password("mariadb_root_password"),
"mariadb_root_password": database_server_doc.get_password("mariadb_root_password"),
}

# offsite config
Expand Down Expand Up @@ -1634,7 +1651,7 @@ def backup_site_files_from_snapshot(
):
from press.press.doctype.site_backup.site_backup import get_backup_bucket

data = {
data: dict[str, Any] = {
"site": site,
"bench": bench,
}
Expand Down
220 changes: 119 additions & 101 deletions press/fixtures/agent_job_type.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1845,107 +1845,107 @@
]
},
{
"disabled_auto_retry": 1,
"docstatus": 0,
"doctype": "Agent Job Type",
"max_retry_count": 3,
"modified": "2025-10-03 18:02:49.847351",
"name": "Change Bench Directory",
"request_method": "POST",
"request_path": "server/change-bench-directory",
"steps": [
{
"step_name": "Change Bench Directory"
},
{
"step_name": "Update Agent Nginx Conf File"
},
{
"step_name": "Update Bench Nginx Conf File"
},
{
"step_name": "Configure Site with Redis Private IP"
},
{
"step_name": "Restart Benches"
}
]
},
{
"disabled_auto_retry": 1,
"docstatus": 0,
"doctype": "Agent Job Type",
"max_retry_count": 3,
"modified": "2025-10-03 18:02:49.847351",
"name": "Stop Bench Workers",
"request_method": "POST",
"request_path": "server/stop-bench-workers",
"steps": [
{
"step_name": "Stop Bench Workers"
}
]
},
{
"disabled_auto_retry": 1,
"docstatus": 0,
"doctype": "Agent Job Type",
"max_retry_count": 3,
"modified": "2025-10-03 18:02:49.847351",
"name": "Add Servers to ACL",
"request_method": "POST",
"request_path": "nfs/add-to-acl",
"steps": [
{
"step_name": "Add Servers to ACL"
}
]
},
{
"disabled_auto_retry": 1,
"docstatus": 0,
"doctype": "Agent Job Type",
"max_retry_count": 3,
"modified": "2025-10-03 18:02:49.847351",
"name": "Remove Servers from ACL",
"request_method": "POST",
"request_path": "nfs/remove-from-acl",
"steps": [
{
"step_name": "Remove Servers from ACL"
}
]
},
{
"disabled_auto_retry": 1,
"docstatus": 0,
"doctype": "Agent Job Type",
"max_retry_count": 3,
"modified": "2025-10-03 18:02:49.847351",
"name": "Start Bench Workers",
"request_method": "POST",
"request_path": "server/start-bench-workers",
"steps": [
{
"step_name": "Start Bench Workers"
}
]
},
{
"disabled_auto_retry": 1,
"docstatus": 0,
"doctype": "Agent Job Type",
"max_retry_count": 3,
"modified": "2025-10-03 18:02:49.847351",
"name": "Force Remove All Benches",
"request_method": "POST",
"request_path": "server/force-remove-all-benches",
"steps": [
{
"step_name": "Force Remove All Benches"
}
]
},
"disabled_auto_retry": 1,
"docstatus": 0,
"doctype": "Agent Job Type",
"max_retry_count": 3,
"modified": "2025-10-03 18:02:49.847351",
"name": "Change Bench Directory",
"request_method": "POST",
"request_path": "server/change-bench-directory",
"steps": [
{
"step_name": "Change Bench Directory"
},
{
"step_name": "Update Agent Nginx Conf File"
},
{
"step_name": "Update Bench Nginx Conf File"
},
{
"step_name": "Configure Site with Redis Private IP"
},
{
"step_name": "Restart Benches"
}
]
},
{
"disabled_auto_retry": 1,
"docstatus": 0,
"doctype": "Agent Job Type",
"max_retry_count": 3,
"modified": "2025-10-03 18:02:49.847351",
"name": "Stop Bench Workers",
"request_method": "POST",
"request_path": "server/stop-bench-workers",
"steps": [
{
"step_name": "Stop Bench Workers"
}
]
},
{
"disabled_auto_retry": 1,
"docstatus": 0,
"doctype": "Agent Job Type",
"max_retry_count": 3,
"modified": "2025-10-03 18:02:49.847351",
"name": "Add Servers to ACL",
"request_method": "POST",
"request_path": "nfs/add-to-acl",
"steps": [
{
"step_name": "Add Servers to ACL"
}
]
},
{
"disabled_auto_retry": 1,
"docstatus": 0,
"doctype": "Agent Job Type",
"max_retry_count": 3,
"modified": "2025-10-03 18:02:49.847351",
"name": "Remove Servers from ACL",
"request_method": "POST",
"request_path": "nfs/remove-from-acl",
"steps": [
{
"step_name": "Remove Servers from ACL"
}
]
},
{
"disabled_auto_retry": 1,
"docstatus": 0,
"doctype": "Agent Job Type",
"max_retry_count": 3,
"modified": "2025-10-03 18:02:49.847351",
"name": "Start Bench Workers",
"request_method": "POST",
"request_path": "server/start-bench-workers",
"steps": [
{
"step_name": "Start Bench Workers"
}
]
},
{
"disabled_auto_retry": 1,
"docstatus": 0,
"doctype": "Agent Job Type",
"max_retry_count": 3,
"modified": "2025-10-03 18:02:49.847351",
"name": "Force Remove All Benches",
"request_method": "POST",
"request_path": "server/force-remove-all-benches",
"steps": [
{
"step_name": "Force Remove All Benches"
}
]
},
{
"disabled_auto_retry": 0,
"docstatus": 0,
Expand Down Expand Up @@ -1990,5 +1990,23 @@
"step_name": "Update Database Host"
}
]
},
{
"disabled_auto_retry": 1,
"docstatus": 0,
"doctype": "Agent Job Type",
"max_retry_count": 3,
"modified": "2025-11-24 16:48:21.908330",
"name": "Update Database Schema Sizes",
"request_method": "POST",
"request_path": "/database/update-schema-sizes",
"steps": [
{
"step_name": "Setup Metadata Table"
},
{
"step_name": "Update Database Schema Size in Metadata Table"
}
]
}
]
3 changes: 3 additions & 0 deletions press/hooks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,9 @@
"*/2 * * * *": [
"press.press.doctype.incident.incident.resolve_incidents",
],
"45 * * * *": [
"press.press.doctype.database_server.database_server.update_database_schema_sizes",
],
"0 4 * * *": [
"press.press.doctype.site.backups.cleanup_offsite",
"press.press.doctype.site.backups.expire_physical",
Expand Down
9 changes: 9 additions & 0 deletions press/playbooks/provide_frappe_user_du_permission.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
---
- name: Provide frappe user du permission
hosts: all
become: yes
become_user: root
gather_facts: yes

roles:
- provide_frappe_user_du_permission
10 changes: 10 additions & 0 deletions press/playbooks/roles/provide_frappe_user_du_permission/tasks/main.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
- name: Allow frappe user to run du
lineinfile:
path: /etc/sudoers.d/frappe
line: 'frappe ALL = (root) NOPASSWD: /usr/bin/du'
create: yes
state: present
validate: 'visudo -cf %s'
mode: '0440'
owner: root
group: root
2 changes: 1 addition & 1 deletion press/playbooks/roles/user/files/sudoers
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,4 @@ frappe ALL = (root) NOPASSWD: /bin/systemctl * alertmanager
frappe ALL = (root) NOPASSWD: /usr/bin/supervisorctl
frappe ALL = (root) NOPASSWD: /usr/sbin/nginx
frappe ALL = (root) NOPASSWD: /usr/local/bin/bench

frappe ALL = (root) NOPASSWD: /usr/bin/du
6 changes: 6 additions & 0 deletions press/press/doctype/database_server/database_server.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,12 @@ frappe.ui.form.on('Database Server', {
true,
frm.doc.is_server_setup,
],
[
'Provide Frappe User DU Permission',
'provide_frappe_user_du_permission',
true,
frm.doc.is_server_setup,
],
].forEach(([label, method, confirm, condition]) => {
if (typeof condition === 'undefined' || condition) {
frm.add_custom_button(
Expand Down
Loading
Loading