Merge pull request #4044 from frappe/mergify/bp/master/pr-4031

Author: tanmoysrt

press/agent.py

@@ -191,7 +191,7 @@ def restore_site(self, site: "Site", skip_failing_patches=False):
 		)
 
 	def rename_site(self, site, new_name: str, create_user: dict | None = None, config: dict | None = None):
-		data = {"new_name": new_name}
+		data: dict[str, Any] = {"new_name": new_name}
 		if create_user:
 			data["create_user"] = create_user
 		if config:
@@ -1034,7 +1034,7 @@ def create_agent_job(
 		self,
 		job_type,
 		path,
-		data=None,
+		data: dict | None = None,
 		files=None,
 		method="POST",
 		bench=None,
@@ -1055,12 +1055,12 @@ def create_agent_job(
 		)
 
 		if not disable_agent_job_deduplication:
-			job = self.get_similar_in_execution_job(
+			existing_job = self.get_similar_in_execution_job(
 				job_type, path, bench, site, code_server, upstream, host, method
 			)
 
-			if job:
-				return job
+			if existing_job:
+				return existing_job
 
 		job: "AgentJob" = frappe.get_doc(
 			{
@@ -1392,6 +1392,23 @@ def kill_database_process(self, site, id):
 			},
 		)
 
+	def update_database_schema_sizes(self):
+		if self.server_type != "Database Server":
+			return NotImplementedError("This method is only supported for Database Server")
+
+		return self.create_agent_job(
+			"Update Database Schema Sizes",
+			"database/update-schema-sizes",
+			data={
+				"private_ip": frappe.get_value("Database Server", self.server, "private_ip"),
+				"mariadb_root_password": get_decrypted_password(
+					"Database Server", self.server, "mariadb_root_password"
+				),
+			},
+			reference_doctype=self.server_type,
+			reference_name=self.server,
+		)
+
 	def fetch_database_variables(self):
 		if self.server_type != "Database Server":
 			return NotImplementedError("Only Database Server supports this method")
@@ -1629,14 +1646,14 @@ def backup_site_database_from_snapshot(
 	):
 		from press.press.doctype.site_backup.site_backup import get_backup_bucket
 
-		database_server: DatabaseServer = frappe.get_doc("Database Server", database_server)
+		database_server_doc: DatabaseServer = frappe.get_doc("Database Server", database_server)  # type: ignore
 		data = {
 			"site": site,
 			"database_name": database_name,
 			"database_ip": frappe.get_value(
-				"Virtual Machine", database_server.virtual_machine, "private_ip_address"
+				"Virtual Machine", database_server_doc.virtual_machine, "private_ip_address"
 			),
-			"mariadb_root_password": database_server.get_password("mariadb_root_password"),
+			"mariadb_root_password": database_server_doc.get_password("mariadb_root_password"),
 		}
 
 		# offsite config
@@ -1672,7 +1689,7 @@ def backup_site_files_from_snapshot(
 	):
 		from press.press.doctype.site_backup.site_backup import get_backup_bucket
 
-		data = {
+		data: dict[str, Any] = {
 			"site": site,
 			"bench": bench,
 		}

press/hooks.py

@@ -262,6 +262,9 @@
 		"*/2 * * * *": [
 			"press.press.doctype.incident.incident.resolve_incidents",
 		],
+		"45 * * * *": [
+			"press.press.doctype.database_server.database_server.update_database_schema_sizes",
+		],
 		"0 4 * * *": [
 			"press.press.doctype.site.backups.cleanup_offsite",
 			"press.press.doctype.site.backups.expire_physical",

press/playbooks/provide_frappe_user_du_permission.yml

@@ -0,0 +1,9 @@
+---
+- name: Provide frappe user du permission
+  hosts: all
+  become: yes
+  become_user: root
+  gather_facts: yes
+
+  roles:
+    - provide_frappe_user_du_permission

press/playbooks/roles/provide_frappe_user_du_permission/tasks/main.yml

@@ -0,0 +1,10 @@
+- name: Allow frappe user to run du
+  lineinfile:
+    path: /etc/sudoers.d/frappe
+    line: 'frappe ALL = (root) NOPASSWD: /usr/bin/du'
+    create: yes
+    state: present
+    validate: 'visudo -cf %s'
+    mode: '0440'
+    owner: root
+    group: root

press/playbooks/roles/user/files/sudoers

@@ -6,4 +6,4 @@ frappe ALL = (root) NOPASSWD: /bin/systemctl * alertmanager
 frappe ALL = (root) NOPASSWD: /usr/bin/supervisorctl
 frappe ALL = (root) NOPASSWD: /usr/sbin/nginx
 frappe ALL = (root) NOPASSWD: /usr/local/bin/bench
-
+frappe ALL = (root) NOPASSWD: /usr/bin/du

press/press/doctype/database_server/database_server.js

@@ -186,6 +186,12 @@ frappe.ui.form.on('Database Server', {
 				true,
 				frm.doc.is_server_setup,
 			],
+			[
+				'Provide Frappe User DU Permission',
+				'provide_frappe_user_du_permission',
+				true,
+				frm.doc.is_server_setup,
+			],
 		].forEach(([label, method, confirm, condition]) => {
 			if (typeof condition === 'undefined' || condition) {
 				frm.add_custom_button(

press/press/doctype/database_server/database_server.py

@@ -1260,6 +1260,7 @@ def _capture_process_list(self) -> str | None:
 			return task.output
 		except Exception:
 			log_error("Process List Capture Exception", server=self.as_dict())
+			return None
 
 	@frappe.whitelist()
 	def setup_pt_stalk(self):
@@ -1311,6 +1312,24 @@ def _setup_logrotate(self):
 		except Exception:
 			log_error("Logrotate Setup Exception", server=self.as_dict())
 
+	@frappe.whitelist()
+	def provide_frappe_user_du_permission(self):
+		frappe.enqueue_doc(
+			self.doctype, self.name, "_provide_frappe_user_du_permission", queue="long", timeout=1200
+		)
+
+	def _provide_frappe_user_du_permission(self):
+		try:
+			ansible = Ansible(
+				playbook="provide_frappe_user_du_permission.yml",
+				server=self,
+				user=self._ssh_user(),
+				port=self._ssh_port(),
+			)
+			ansible.run()
+		except Exception:
+			log_error("MariaDB Provide Frappe User DU Permission Exception", server=self.as_dict())
+
 	@frappe.whitelist()
 	def setup_mariadb_debug_symbols(self):
 		frappe.enqueue_doc(
@@ -2013,8 +2032,8 @@ def remove_binlogs_from_indexer(self):
 		# Generate series of binlog
 		unindexable_binlogs = []
 		for binlog_no in range(first_binlog_no, last_binlog_no + 1):
-			binlog_no = str(binlog_no).zfill(no_of_digits)
-			unindexable_binlogs.append(f"mysql-bin.{binlog_no}")
+			binlog_no_z_filled = str(binlog_no).zfill(no_of_digits)
+			unindexable_binlogs.append(f"mysql-bin.{binlog_no_z_filled}")
 
 		self.agent.remove_binlogs_from_indexer(unindexable_binlogs)
 
@@ -2192,6 +2211,9 @@ def _set_mariadb_mount_dependency(self):
 		except Exception:
 			log_error("Set MariaDB Mount Dependency Exception", server=self.as_dict())
 
+	def update_database_schema_sizes(self):
+		self.agent.update_database_schema_sizes()
+
 
 get_permission_query_conditions = get_permission_query_conditions_for_doctype("Database Server")
 
@@ -2420,3 +2442,26 @@ def auto_purge_binlogs_by_size_limit():
 			return
 		except Exception:
 			frappe.db.rollback()
+
+
+def update_database_schema_sizes():
+	databases = frappe.db.get_all(
+		"Database Server",
+		filters={
+			"status": "Active",
+		},
+		pluck="name",
+	)
+
+	for database in databases:
+		if has_job_timeout_exceeded():
+			return
+		try:
+			server: DatabaseServer = frappe.get_doc("Database Server", database)
+			server.update_database_schema_sizes()
+			frappe.db.commit()
+		except rq.timeouts.JobTimeoutException:
+			frappe.db.rollback()
+			return
+		except Exception:
+			frappe.db.rollback()

press/press/doctype/server/server.py

@@ -65,7 +65,7 @@ class BenchInfoType(TypedDict):
 
 
 class ARMDockerImageType(TypedDict):
-	build: str
+	build: str | None
 	status: Literal["Pending", "Preparing", "Running", "Failure", "Success"]
 	bench: str
 
@@ -1856,12 +1856,12 @@ def size_to_increase_by_for_20_percent_available(self, mountpoint: str):  # min
 
 		return int(max(self.auto_add_storage_min, min(projected_growth_gb, self.auto_add_storage_max)))
 
-	def recommend_disk_increase(self, mountpoint: str | None = None):
+	def recommend_disk_increase(self, mountpoint: str):
 		"""
 		Send disk expansion email to users with disabled auto addon storage at 80% capacity
 		Calculate the disk usage over a 30 hour period and take 25 percent of that
 		"""
-		server: Server | DatabaseServer = frappe.get_doc(self.doctype, self.name)
+		server: Server | DatabaseServer = frappe.get_doc(self.doctype, self.name)  # type: ignore
 		if server.auto_increase_storage:
 			return
 
@@ -1875,26 +1875,26 @@ def recommend_disk_increase(self, mountpoint: str | None = None):
 			/ 1024
 		)
 
-		current_disk_usage = current_disk_usage / 1024 / 1024 / 1024
-		disk_capacity = disk_capacity / 1024 / 1024 / 1024
+		current_disk_usage_flt = round(current_disk_usage / 1024 / 1024 / 1024, 2)
+		disk_capacity_flt = round(disk_capacity / 1024 / 1024 / 1024, 2)
 
 		frappe.sendmail(
 			recipients=get_communication_info("Email", "Incident", self.doctype, self.name),
 			subject=f"Important: Server {server.name} has used 80% of the available space",
 			template="disabled_auto_disk_expansion",
 			args={
 				"server": server.name,
-				"current_disk_usage": f"{current_disk_usage} Gib",
-				"available_disk_space": f"{disk_capacity} GiB",
+				"current_disk_usage": f"{current_disk_usage_flt} Gib",
+				"available_disk_space": f"{disk_capacity_flt} GiB",
 				"used_storage_percentage": "80%",
 				"increase_by": f"{recommended_increase} GiB",
 			},
 		)
 
 	def calculated_increase_disk_size(
 		self,
+		mountpoint: str,
 		additional: int = 0,
-		mountpoint: str | None = None,
 	):
 		"""
 		Calculate required disk increase for servers and handle notifications accordingly.
@@ -1913,10 +1913,8 @@ def calculated_increase_disk_size(
 		buffer = self.size_to_increase_by_for_20_percent_available(mountpoint)
 		server: Server | DatabaseServer = frappe.get_doc(self.doctype, self.name)
 		disk_capacity = self.disk_capacity(mountpoint)
-		current_disk_usage = disk_capacity - self.free_space(mountpoint)
 
-		current_disk_usage = round(current_disk_usage / 1024 / 1024 / 1024, 2)
-		disk_capacity = round(disk_capacity / 1024 / 1024 / 1024, 2)
+		current_disk_usage = round((disk_capacity - self.free_space(mountpoint)) / 1024 / 1024 / 1024, 2)
 
 		if not server.auto_increase_storage and (not server.has_data_volume or mountpoint != "/"):
 			TelegramMessage.enqueue(
@@ -2087,6 +2085,7 @@ def set_additional_config(self):  # noqa: C901
 
 		if self.doctype == "Database Server":
 			self.adjust_memory_config()
+			self.provide_frappe_user_du_permission()
 			self.setup_logrotate()
 			self.setup_user_lingering()