From 02c610544444be3e54ad0347ed9a9d1ebc44e520 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 21 Feb 2022 17:35:31 +0530 Subject: [PATCH 01/20] fix(email_account): Decode uid from bytes before update --- frappe/email/doctype/email_account/email_account.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index 992c7a88b470..641b3e2530a0 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -417,7 +417,13 @@ def insert_communication(self, msg, args=None): if names: name = names[0].get("name") # email is already available update communication uid instead - frappe.db.set_value("Communication", name, "uid", uid, update_modified=False) + frappe.db.set_value( + "Communication", + name, + "uid", + frappe.safe_decode(uid), + update_modified=False, + ) self.flags.notify = False From ca2d987ab7b35d159a838700ef684056b7679601 Mon Sep 17 00:00:00 2001 From: Shariq Ansari <30859809+shariquerik@users.noreply.github.com> Date: Fri, 27 May 2022 21:06:20 +0530 Subject: [PATCH 02/20] fix: Strip all spacing characters from Message-ID & In-Reply-To (backport #16999) (#17004) --- frappe/core/doctype/communication/email.py | 58 +++++++++++-------- .../doctype/email_account/email_account.py | 19 +++++- .../email_account/test_email_account.py | 2 +- frappe/email/queue.py | 19 +++++- frappe/email/receive.py | 23 ++++++-- frappe/utils/data.py | 11 ++++ 6 files changed, 97 insertions(+), 35 deletions(-) diff --git a/frappe/core/doctype/communication/email.py b/frappe/core/doctype/communication/email.py index ee7acd749273..ec7c04d3c5f5 100755 --- a/frappe/core/doctype/communication/email.py +++ b/frappe/core/doctype/communication/email.py @@ -11,9 +11,17 @@ from frappe.utils import (get_url, get_formatted_email, cint, list_to_str, validate_email_address, split_emails, parse_addr, get_datetime) from frappe.email.email_body import get_message_id -import frappe.email.smtp -import time -from frappe import _ +from frappe.utils import ( + cint, + get_datetime, + get_formatted_email, + get_string_between, + get_url, + list_to_str, + parse_addr, + split_emails, + validate_email_address, +) from frappe.utils.background_jobs import enqueue @frappe.whitelist() @@ -55,27 +63,29 @@ def make(doctype=None, name=None, content=None, subject=None, sent_or_received = cc = list_to_str(cc) if isinstance(cc, list) else cc bcc = list_to_str(bcc) if isinstance(bcc, list) else bcc - comm = frappe.get_doc({ - "doctype":"Communication", - "subject": subject, - "content": content, - "sender": sender, - "sender_full_name":sender_full_name, - "recipients": recipients, - "cc": cc or None, - "bcc": bcc or None, - "communication_medium": communication_medium, - "sent_or_received": sent_or_received, - "reference_doctype": doctype, - "reference_name": name, - "email_template": email_template, - "message_id":get_message_id().strip(" <>"), - "read_receipt":read_receipt, - "has_attachment": 1 if attachments else 0, - "communication_type": communication_type - }).insert(ignore_permissions=True) - - comm.save(ignore_permissions=True) + comm = frappe.get_doc( + { + "doctype": "Communication", + "subject": subject, + "content": content, + "sender": sender, + "sender_full_name": sender_full_name, + "recipients": recipients, + "cc": cc or None, + "bcc": bcc or None, + "communication_medium": communication_medium, + "sent_or_received": sent_or_received, + "reference_doctype": doctype, + "reference_name": name, + "email_template": email_template, + "message_id": get_string_between("<", get_message_id(), ">"), + "read_receipt": read_receipt, + "has_attachment": 1 if attachments else 0, + "communication_type": communication_type, + } + ) + comm.flags.skip_add_signature = not add_signature + comm.insert(ignore_permissions=True) if isinstance(attachments, string_types): attachments = json.loads(attachments) diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index 641b3e2530a0..e632b5d56729 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -10,8 +10,20 @@ import time from frappe import _, safe_encode from frappe.model.document import Document -from frappe.utils import validate_email_address, cint, cstr, get_datetime, DATE_FORMAT, strip, comma_or, sanitize_html, add_days -from frappe.utils.user import is_system_user +from frappe.utils import ( + DATE_FORMAT, + add_days, + cint, + comma_or, + cstr, + get_datetime, + get_string_between, + sanitize_html, + strip, + validate_email_address, +) +from frappe.utils.background_jobs import enqueue, get_jobs +from frappe.utils.html_utils import clean_email_html from frappe.utils.jinja import render_template from frappe.email.smtp import SMTPServer from frappe.email.receive import EmailServer, Email @@ -608,7 +620,8 @@ def find_parent_from_in_reply_to(self, communication, email): Message-ID is formatted as `{message_id}@{site}`''' parent = None - in_reply_to = (email.mail.get("In-Reply-To") or "").strip(" <>") + in_reply_to = email.mail.get("In-Reply-To") or "" + in_reply_to = get_string_between("<", in_reply_to, ">") if in_reply_to: if "@{0}".format(frappe.local.site) in in_reply_to: diff --git a/frappe/email/doctype/email_account/test_email_account.py b/frappe/email/doctype/email_account/test_email_account.py index f87ee32bb129..542c1bdc6100 100644 --- a/frappe/email/doctype/email_account/test_email_account.py +++ b/frappe/email/doctype/email_account/test_email_account.py @@ -186,7 +186,7 @@ def test_threading_by_message_id(self): # get test mail with message-id as in-reply-to with open(os.path.join(os.path.dirname(__file__), "test_mails", "reply-4.raw"), "r") as f: - test_mails = [f.read().replace('{{ message_id }}', last_mail.message_id)] + test_mails = [f.read().replace("{{ message_id }}", "<" + last_mail.message_id + ">")] # pull the mail email_account = frappe.get_doc("Email Account", "_Test Email Account 1") diff --git a/frappe/email/queue.py b/frappe/email/queue.py index a0c029f5380d..4a6441272134 100755 --- a/frappe/email/queue.py +++ b/frappe/email/queue.py @@ -17,7 +17,22 @@ from email.parser import Parser -class EmailLimitCrossedError(frappe.ValidationError): pass +import frappe +from frappe import _, enqueue, msgprint, safe_decode, safe_encode +from frappe.email.email_body import add_attachment, get_email, get_formatted_html +from frappe.email.smtp import SMTPServer, get_outgoing_email_account +from frappe.utils import ( + add_days, + cint, + cstr, + get_hook_method, + get_string_between, + get_url, + now_datetime, + nowdate, + split_emails, +) +from frappe.utils.verified_command import get_signed_params, verify_request def send(recipients=None, sender=None, subject=None, message=None, text_content=None, reference_doctype=None, reference_name=None, unsubscribe_method=None, unsubscribe_params=None, unsubscribe_message=None, @@ -225,7 +240,7 @@ def get_email_queue(recipients, sender, subject, **kwargs): if kwargs.get('in_reply_to'): mail.set_in_reply_to(kwargs.get('in_reply_to')) - e.message_id = mail.msg_root["Message-Id"].strip(" <>") + e.message_id = get_string_between("<", mail.msg_root["Message-Id"], ">") e.message = cstr(mail.as_string()) e.sender = mail.sender diff --git a/frappe/email/receive.py b/frappe/email/receive.py index 6d60007cdbab..e831ee10033e 100644 --- a/frappe/email/receive.py +++ b/frappe/email/receive.py @@ -17,10 +17,21 @@ import frappe from frappe import _, safe_decode, safe_encode -from frappe.core.doctype.file.file import (MaxFileSizeReachedError, - get_random_filename) -from frappe.utils import (cint, convert_utc_to_user_timezone, cstr, - extract_email_id, markdown, now, parse_addr, strip) +from frappe.core.doctype.file.file import MaxFileSizeReachedError, get_random_filename +from frappe.utils import ( + cint, + convert_utc_to_user_timezone, + cstr, + extract_email_id, + get_string_between, + markdown, + now, + parse_addr, + strip, +) + +# fix due to a python bug in poplib that limits it to 2048 +poplib._MAXLINE = 20480 class EmailSizeExceededError(frappe.ValidationError): pass @@ -377,7 +388,9 @@ def __init__(self, content): self.set_content_and_type() self.set_subject() self.set_from() - self.message_id = (self.mail.get('Message-ID') or "").strip(" <>") + + message_id = self.mail.get("Message-ID") or "" + self.message_id = get_string_between("<", message_id, ">") if self.mail["Date"]: try: diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 60f0b361e76c..4cfdbfd2540d 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -1457,6 +1457,17 @@ def strip(val, chars=None): # \ufeff is no-width-break, \u200b is no-width-space return (val or "").replace("\ufeff", "").replace("\u200b", "").strip(chars) + +def get_string_between(start, string, end): + if not string: + return "" + + regex = "{0}(.*){1}".format(start, end) + out = re.search(regex, string) + + return out.group(1) if out else string + + def to_markdown(html): from html2text import html2text from six.moves import html_parser as HTMLParser From f7df0aa5bd69d99ef600b4853ad20d3ab9d2ed9a Mon Sep 17 00:00:00 2001 From: "FinByz Tech Pvt. Ltd" Date: Mon, 30 May 2022 16:27:45 +0530 Subject: [PATCH 03/20] Update email_account.py If reference_doctype and reference_name is getting None then Communication will not link with append_to DocType. --- frappe/email/doctype/email_account/email_account.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index e632b5d56729..2601436c00ff 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -647,12 +647,13 @@ def find_parent_from_in_reply_to(self, communication, email): parent = frappe.get_doc(parent.reference_doctype, parent.reference_name) else: - comm = frappe.db.get_value('Communication', - dict( - message_id=in_reply_to, - creation=['>=', add_days(get_datetime(), -30)]), - ['reference_doctype', 'reference_name'], as_dict=1) - if comm: + comm = frappe.db.get_value( + "Communication", + dict(message_id=in_reply_to, creation=[">=", add_days(get_datetime(), -30)]), + ["reference_doctype", "reference_name"], + as_dict=1, + ) + if comm and comm.reference_doctype and comm.reference_name: parent = frappe._dict(doctype=comm.reference_doctype, name=comm.reference_name) return parent From 24a5c82bd6cad0fe4540882e67f4de55ea38ce4d Mon Sep 17 00:00:00 2001 From: skjbulcher Date: Fri, 1 Jul 2022 02:22:43 -0700 Subject: [PATCH 04/20] fix: don't try appending email to sent folder if enable incoming and imap is turned off (#17344) This can happen if email domain is updated - which forcibly updates settings of all email accounts associated with that email domain --- frappe/email/doctype/email_account/email_account.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index 2601436c00ff..96daa23d2f04 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -754,6 +754,12 @@ def check_automatic_linking_email_account(self): def append_email_to_sent_folder(self, message): + if not (self.enable_incoming and self.use_imap): + # don't try appending if enable incoming and imap is not set + # as email domain's updation can cause email account(s) to forcibly + # update their settings. + return + email_server = None try: email_server = self.get_incoming_server(in_receive=True) From 8b1f0ce38ce7d7adf84082affb876713361502e8 Mon Sep 17 00:00:00 2001 From: HENRY Florian Date: Mon, 22 Aug 2022 15:32:27 +0200 Subject: [PATCH 05/20] feat: STARTTLS authentication for IMAP (backport #17683) (#17881) --- .../doctype/email_account/email_account.json | 11 +++- .../doctype/email_account/email_account.py | 51 ++++++++++++------- .../doctype/email_domain/email_domain.json | 10 +++- .../doctype/email_domain/email_domain.py | 35 ++++++++++--- .../doctype/email_domain/test_email_domain.py | 1 + frappe/email/receive.py | 7 ++- frappe/translations/fr.csv | 1 + 7 files changed, 88 insertions(+), 28 deletions(-) diff --git a/frappe/email/doctype/email_account/email_account.json b/frappe/email/doctype/email_account/email_account.json index ca92d39b022b..14c6b8e68a88 100644 --- a/frappe/email/doctype/email_account/email_account.json +++ b/frappe/email/doctype/email_account/email_account.json @@ -25,6 +25,7 @@ "default_incoming", "use_imap", "use_ssl", + "use_starttls", "email_server", "incoming_port", "column_break_18", @@ -562,12 +563,20 @@ "fieldname": "account_section", "fieldtype": "Section Break", "label": "Account" + }, + { + "default": "0", + "depends_on": "eval:!doc.domain && doc.enable_incoming && doc.use_imap && !doc.use_ssl", + "fetch_from": "domain.use_starttls", + "fieldname": "use_starttls", + "fieldtype": "Check", + "label": "Use STARTTLS" } ], "icon": "fa fa-inbox", "index_web_pages_for_search": 1, "links": [], - "modified": "2021-09-22 12:11:15.603556", + "modified": "2021-09-22 13:11:15.603556", "modified_by": "Administrator", "module": "Email", "name": "Email Account", diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index 96daa23d2f04..fbbd85dfc102 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -77,9 +77,14 @@ def validate(self): #if self.enable_incoming and not self.append_to: # frappe.throw(_("Append To is mandatory for incoming mails")) - if (not self.awaiting_password and not frappe.local.flags.in_install - and not frappe.local.flags.in_patch): - if self.password or self.smtp_server in ('127.0.0.1', 'localhost'): + self.use_starttls = cint(self.use_imap and self.use_starttls and not self.use_ssl) + + if ( + not self.awaiting_password + and not frappe.local.flags.in_install + and not frappe.local.flags.in_patch + ): + if self.password or self.smtp_server in ("127.0.0.1", "localhost"): if self.enable_incoming: self.get_incoming_server() self.no_failed = 0 @@ -152,10 +157,17 @@ def get_domain(self, email_id): try: domain = email_id.split("@") fields = [ - "name as domain", "use_imap", "email_server", - "use_ssl", "smtp_server", "use_tls", - "smtp_port", "incoming_port", "append_emails_to_sent_folder", - "use_ssl_for_outgoing" + "name as domain", + "use_imap", + "email_server", + "use_ssl", + "use_starttls", + "smtp_server", + "use_tls", + "smtp_port", + "incoming_port", + "append_emails_to_sent_folder", + "use_ssl_for_outgoing", ] return frappe.db.get_value("Email Domain", domain[1], fields, as_dict=True) except Exception: @@ -184,17 +196,20 @@ def get_incoming_server(self, in_receive=False, email_sync_rule="UNSEEN"): if frappe.cache().get_value("workers:no-internet") == True: return None - args = frappe._dict({ - "email_account": self.name, - "host": self.email_server, - "use_ssl": self.use_ssl, - "username": getattr(self, "login_id", None) or self.email_id, - "use_imap": self.use_imap, - "email_sync_rule": email_sync_rule, - "uid_validity": self.uidvalidity, - "incoming_port": get_port(self), - "initial_sync_count": self.initial_sync_count or 100 - }) + args = frappe._dict( + { + "email_account": self.name, + "host": self.email_server, + "use_ssl": self.use_ssl, + "use_starttls": self.use_starttls, + "username": getattr(self, "login_id", None) or self.email_id, + "use_imap": self.use_imap, + "email_sync_rule": email_sync_rule, + "uid_validity": self.uidvalidity, + "incoming_port": get_port(self), + "initial_sync_count": self.initial_sync_count or 100, + } + ) if self.password: args.password = self.get_password() diff --git a/frappe/email/doctype/email_domain/email_domain.json b/frappe/email/doctype/email_domain/email_domain.json index a4ca19a0bd1f..7a1e40d1d57e 100644 --- a/frappe/email/doctype/email_domain/email_domain.json +++ b/frappe/email/doctype/email_domain/email_domain.json @@ -13,6 +13,7 @@ "email_server", "use_imap", "use_ssl", + "use_starttls", "incoming_port", "attachment_limit", "append_to", @@ -121,11 +122,18 @@ "fieldname": "use_ssl_for_outgoing", "fieldtype": "Check", "label": "Use SSL for Outgoing" + }, + { + "default": "0", + "depends_on": "eval:doc.use_imap && !doc.use_ssl", + "fieldname": "use_starttls", + "fieldtype": "Check", + "label": "Use STARTTLS" } ], "icon": "icon-inbox", "links": [], - "modified": "2019-12-18 15:57:34.445308", + "modified": "2022-08-19 11:22:46.342609", "modified_by": "Administrator", "module": "Email", "name": "Email Domain", diff --git a/frappe/email/doctype/email_domain/email_domain.py b/frappe/email/doctype/email_domain/email_domain.py index ce3952356401..bb4518ba46eb 100644 --- a/frappe/email/doctype/email_domain/email_domain.py +++ b/frappe/email/doctype/email_domain/email_domain.py @@ -31,6 +31,7 @@ def validate(self): logger.info('Checking incoming IMAP email server {host}:{port} ssl={ssl}...'.format( host=self.email_server, port=get_port(self), ssl=self.use_ssl)) if self.use_ssl: + self.use_starttls = 0 test = imaplib.IMAP4_SSL(self.email_server, port=get_port(self)) else: test = imaplib.IMAP4(self.email_server, port=get_port(self)) @@ -44,9 +45,13 @@ def validate(self): test = poplib.POP3(self.email_server, port=get_port(self)) except Exception as e: - logger.warn('Incoming email account "{host}" not correct'.format(host=self.email_server), exc_info=e) - frappe.throw(title=_("Incoming email account not correct"), - msg='Error connecting IMAP/POP3 "{host}": {e}'.format(host=self.email_server, e=e)) + logger.warning( + 'Incoming email account "{host}" not correct'.format(host=self.email_server), exc_info=e + ) + frappe.throw( + title=_("Incoming email account not correct"), + msg='Error connecting IMAP/POP3 "{host}": {e}'.format(host=self.email_server, e=e), + ) finally: try: @@ -74,16 +79,32 @@ def validate(self): sess = smtplib.SMTP(cstr(self.smtp_server or ""), cint(self.smtp_port) or None) sess.quit() except Exception as e: - logger.warn('Outgoing email account "{host}" not correct'.format(host=self.smtp_server), exc_info=e) - frappe.throw(title=_("Outgoing email account not correct"), - msg='Error connecting SMTP "{host}": {e}'.format(host=self.smtp_server, e=e)) + logger.warning( + 'Outgoing email account "{host}" not correct'.format(host=self.smtp_server), exc_info=e + ) + frappe.throw( + title=_("Outgoing email account not correct"), + msg='Error connecting SMTP "{host}": {e}'.format(host=self.smtp_server, e=e), + ) def on_update(self): """update all email accounts using this domain""" for email_account in frappe.get_all("Email Account", filters={"domain": self.name}): try: email_account = frappe.get_doc("Email Account", email_account.name) - for attr in ["email_server", "use_imap", "use_ssl", "use_tls", "attachment_limit", "smtp_server", "smtp_port", "use_ssl_for_outgoing", "append_emails_to_sent_folder", "incoming_port"]: + for attr in [ + "email_server", + "use_imap", + "use_ssl", + "use_tls", + "use_starttls", + "attachment_limit", + "smtp_server", + "smtp_port", + "use_ssl_for_outgoing", + "append_emails_to_sent_folder", + "incoming_port", + ]: email_account.set(attr, self.get(attr, default=0)) email_account.save() diff --git a/frappe/email/doctype/email_domain/test_email_domain.py b/frappe/email/doctype/email_domain/test_email_domain.py index 1c5306e9c237..cf0487f237f5 100644 --- a/frappe/email/doctype/email_domain/test_email_domain.py +++ b/frappe/email/doctype/email_domain/test_email_domain.py @@ -34,6 +34,7 @@ def test_on_update(self): self.assertEqual(mail_account.use_imap, mail_domain.use_imap) self.assertEqual(mail_account.use_ssl, mail_domain.use_ssl) self.assertEqual(mail_account.use_tls, mail_domain.use_tls) + self.assertEqual(mail_account.use_starttls, mail_domain.use_starttls) self.assertEqual(mail_account.attachment_limit, mail_domain.attachment_limit) self.assertEqual(mail_account.smtp_server, mail_domain.smtp_server) self.assertEqual(mail_account.smtp_port, mail_domain.smtp_port) diff --git a/frappe/email/receive.py b/frappe/email/receive.py index e831ee10033e..7f254e8884fb 100644 --- a/frappe/email/receive.py +++ b/frappe/email/receive.py @@ -69,7 +69,12 @@ def connect_imap(self): if cint(self.settings.use_ssl): self.imap = Timed_IMAP4_SSL(self.settings.host, self.settings.incoming_port, timeout=frappe.conf.get("pop_timeout")) else: - self.imap = Timed_IMAP4(self.settings.host, self.settings.incoming_port, timeout=frappe.conf.get("pop_timeout")) + self.imap = Timed_IMAP4( + self.settings.host, self.settings.incoming_port, timeout=frappe.conf.get("pop_timeout") + ) + if self.settings.use_starttls: + self.imap.starttls() + self.imap.login(self.settings.username, self.settings.password) # connection established! return True diff --git a/frappe/translations/fr.csv b/frappe/translations/fr.csv index 511c590a59e2..b0250e8b6467 100644 --- a/frappe/translations/fr.csv +++ b/frappe/translations/fr.csv @@ -2639,6 +2639,7 @@ Use IMAP,Utiliser IMAP, Use POST,Utiliser le POST, Use SSL,Utiliser SSL, Use TLS,Utiliser TLS, +Use STARTTLS,Utiliser STARTTLS "Use a few words, avoid common phrases.","Utiliser quelques mots, éviter les phrases courantes.", Use of sub-query or function is restricted,L'utilisation de la sous-requête ou de la fonction est restreinte, Use socketio to upload file,Utilisez socketio pour télécharger le fichier, From edf8ee2498e682b7bdc18084eb7d74c5bb9dfeff Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Thu, 20 Apr 2023 21:48:11 +0530 Subject: [PATCH 06/20] feat: email oauth (v13) (#20790) (#20800) (cherry picked from commit b4b6b668ac5f63652b133c4c02e95572533dd5a7) Co-authored-by: Ritwik Puri --- frappe/core/doctype/user/user.py | 5 -- .../doctype/email_account/email_account.js | 46 ++++++++++-- .../doctype/email_account/email_account.json | 40 +++++++++- .../doctype/email_account/email_account.py | 60 +++++++++++---- frappe/email/oauth.py | 73 +++++++++++++++++++ frappe/email/queue.py | 21 +++--- frappe/email/receive.py | 31 ++++++-- frappe/email/smtp.py | 69 +++++++++--------- .../doctype/connected_app/connected_app.py | 43 +++++++++-- .../doctype/token_cache/token_cache.json | 5 +- .../doctype/token_cache/token_cache.py | 22 ++++-- 11 files changed, 319 insertions(+), 96 deletions(-) create mode 100644 frappe/email/oauth.py diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index 6bed8da31cf6..7232f9610099 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -767,11 +767,6 @@ def get_email_awaiting(user): and parent = %(user)s""", {"user":user}, as_dict=1) if waiting: return waiting - else: - frappe.db.sql("""update `tabUser Email` - set awaiting_password =0 - where parent = %(user)s""",{"user":user}) - return False def ask_pass_update(): # update the sys defaults as to awaiting users diff --git a/frappe/email/doctype/email_account/email_account.js b/frappe/email/doctype/email_account/email_account.js index 1e8030604b1d..8f00d4b8a708 100644 --- a/frappe/email/doctype/email_account/email_account.js +++ b/frappe/email/doctype/email_account/email_account.js @@ -76,7 +76,6 @@ frappe.ui.form.on("Email Account", { frm.set_value(key, value); }); } - frm.events.show_gmail_message_for_less_secure_apps(frm); }, use_imap: function(frm) { @@ -109,13 +108,13 @@ frappe.ui.form.on("Email Account", { onload: function(frm) { frm.set_df_property("append_to", "only_select", true); frm.set_query("append_to", "frappe.email.doctype.email_account.email_account.get_append_to"); + frm.events.show_oauth_authorization_message(frm); }, refresh: function(frm) { frm.events.set_domain_fields(frm); frm.events.enable_incoming(frm); frm.events.notify_if_unreplied(frm); - frm.events.show_gmail_message_for_less_secure_apps(frm); if(frappe.route_flags.delete_user_from_locals && frappe.route_flags.linked_user) { delete frappe.route_flags.delete_user_from_locals; @@ -123,10 +122,45 @@ frappe.ui.form.on("Email Account", { } }, - show_gmail_message_for_less_secure_apps: function(frm) { - frm.dashboard.clear_headline(); - if(frm.doc.service==="GMail") { - frm.dashboard.set_headline_alert('Gmail sólo funcionará si permites el acceso a aplicaciones menos seguras en la configuración de Gmail. Lee esto para más detalles'); + authorize_api_access: function (frm) { + frm.events.oauth_access(frm); + }, + + oauth_access: function(frm) { + frappe.model.with_doc("Connected App", frm.doc.connected_app, () => { + const connected_app = frappe.get_doc("Connected App", frm.doc.connected_app); + return frappe.call({ + doc: connected_app, + method: "initiate_web_application_flow", + args: { + success_uri: window.location.pathname, + user: frm.doc.connected_user, + }, + callback: function (r) { + window.open(r.message, "_self"); + }, + }); + }); + }, + + show_oauth_authorization_message(frm) { + if (frm.doc.auth_method === "OAuth") { + frappe.call({ + method: "frappe.integrations.doctype.connected_app.connected_app.has_token", + args: { + connected_app: frm.doc.connected_app, + connected_user: frm.doc.connected_user, + }, + callback: (r) => { + if (!r.message) { + let msg = __( + 'OAuth has been enabled but not authorised. Please use "Authorise API Access" button to do the same.' + ); + frm.dashboard.clear_headline(); + frm.dashboard.set_headline_alert(msg, "yellow"); + } + }, + }); } }, diff --git a/frappe/email/doctype/email_account/email_account.json b/frappe/email/doctype/email_account/email_account.json index 14c6b8e68a88..0f2e0d9bcbb3 100644 --- a/frappe/email/doctype/email_account/email_account.json +++ b/frappe/email/doctype/email_account/email_account.json @@ -14,10 +14,14 @@ "domain", "service", "authentication_column", + "auth_method", "password", "awaiting_password", "ascii_encode_password", + "authorize_api_access", "column_break_10", + "connected_app", + "connected_user", "login_id_is_different", "login_id", "mailbox_settings", @@ -97,6 +101,7 @@ "label": "Email Login ID" }, { + "depends_on": "eval: doc.auth_method === \"Basic\"", "fieldname": "password", "fieldtype": "Password", "hide_days": 1, @@ -105,6 +110,7 @@ }, { "default": "0", + "depends_on": "eval: doc.auth_method === \"Basic\"", "fieldname": "awaiting_password", "fieldtype": "Check", "hide_days": 1, @@ -113,6 +119,7 @@ }, { "default": "0", + "depends_on": "eval: doc.auth_method === \"Basic\"", "fieldname": "ascii_encode_password", "fieldtype": "Check", "hide_days": 1, @@ -571,12 +578,41 @@ "fieldname": "use_starttls", "fieldtype": "Check", "label": "Use STARTTLS" + }, + { + "default": "Basic", + "fieldname": "auth_method", + "fieldtype": "Select", + "label": "Method", + "options": "Basic\nOAuth" + }, + { + "depends_on": "eval: doc.auth_method === \"OAuth\"", + "fieldname": "connected_app", + "fieldtype": "Link", + "label": "Connected App", + "mandatory_depends_on": "eval: doc.auth_method === \"OAuth\"", + "options": "Connected App" + }, + { + "depends_on": "eval: doc.auth_method === \"OAuth\"", + "fieldname": "connected_user", + "fieldtype": "Link", + "label": "Connected User", + "mandatory_depends_on": "eval: doc.auth_method === \"OAuth\"", + "options": "User" + }, + { + "depends_on": "eval: doc.auth_method === \"OAuth\" && !doc.__islocal && !doc.__unsaved", + "fieldname": "authorize_api_access", + "fieldtype": "Button", + "label": "Authorize API Access" } ], "icon": "fa fa-inbox", "index_web_pages_for_search": 1, "links": [], - "modified": "2021-09-22 13:11:15.603556", + "modified": "2021-09-23 13:11:15.603556", "modified_by": "Administrator", "module": "Email", "name": "Email Account", @@ -598,4 +634,4 @@ "sort_field": "modified", "sort_order": "DESC", "track_changes": 1 -} +} \ No newline at end of file diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index fbbd85dfc102..43187b16cbaa 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -74,17 +74,18 @@ def validate(self): if frappe.local.flags.in_patch or frappe.local.flags.in_test: return - #if self.enable_incoming and not self.append_to: - # frappe.throw(_("Append To is mandatory for incoming mails")) - + use_oauth = self.auth_method == "OAuth" self.use_starttls = cint(self.use_imap and self.use_starttls and not self.use_ssl) - if ( - not self.awaiting_password - and not frappe.local.flags.in_install - and not frappe.local.flags.in_patch - ): - if self.password or self.smtp_server in ("127.0.0.1", "localhost"): + validate_oauth = False + if use_oauth: + # no need for awaiting password for oauth + self.awaiting_password = 0 + self.password = None + validate_oauth = not (self.is_new() and not self.get_oauth_token()) + + if not self.awaiting_password and not frappe.local.flags.in_install: + if validate_oauth or self.password or self.smtp_server in ("127.0.0.1", "localhost"): if self.enable_incoming: self.get_incoming_server() self.no_failed = 0 @@ -94,7 +95,8 @@ def validate(self): self.check_smtp() else: if self.enable_incoming or (self.enable_outgoing and not self.no_smtp_authentication): - frappe.throw(_("Password is required or select Awaiting Password")) + if not use_oauth: + frappe.throw(_("Password is required or select Awaiting Password")) if self.notify_if_unreplied: if not self.send_notification_to: @@ -179,12 +181,15 @@ def check_smtp(self): if not self.smtp_server: frappe.throw(_("{0} is required").format("SMTP Server")) + oauth_token = self.get_oauth_token() server = SMTPServer( login = getattr(self, "login_id", None) or self.email_id, server=self.smtp_server, port=cint(self.smtp_port), use_tls=cint(self.use_tls), - use_ssl=cint(self.use_ssl_for_outgoing) + use_ssl=cint(self.use_ssl_for_outgoing), + use_oauth=self.auth_method == "OAuth", + access_token=oauth_token.get_password("access_token") if oauth_token else None, ) if self.password and not self.no_smtp_authentication: server.password = self.get_password() @@ -196,6 +201,7 @@ def get_incoming_server(self, in_receive=False, email_sync_rule="UNSEEN"): if frappe.cache().get_value("workers:no-internet") == True: return None + oauth_token = self.get_oauth_token() args = frappe._dict( { "email_account": self.name, @@ -208,6 +214,8 @@ def get_incoming_server(self, in_receive=False, email_sync_rule="UNSEEN"): "uid_validity": self.uidvalidity, "incoming_port": get_port(self), "initial_sync_count": self.initial_sync_count or 100, + "use_oauth": self.auth_method == "OAuth", + "access_token": oauth_token.get_password("access_token") if oauth_token else None, } ) @@ -793,6 +801,12 @@ def append_email_to_sent_folder(self, message): except Exception: frappe.log_error() + def get_oauth_token(self): + if self.auth_method == "OAuth": + connected_app = frappe.get_doc("Connected App", self.connected_app) + return connected_app.get_active_token(self.connected_user) + + @frappe.whitelist() def get_append_to(doctype=None, txt=None, searchfield=None, start=None, page_len=None, filters=None): txt = txt if txt else "" @@ -856,14 +870,27 @@ def notify_unreplied(): def pull(now=False): """Will be called via scheduler, pull emails from all enabled Email accounts.""" + from frappe.integrations.doctype.connected_app.connected_app import has_token + if frappe.cache().get_value("workers:no-internet") == True: if test_internet(): frappe.cache().set_value("workers:no-internet", False) else: return - queued_jobs = get_jobs(site=frappe.local.site, key='job_name')[frappe.local.site] - for email_account in frappe.get_list("Email Account", - filters={"enable_incoming": 1, "awaiting_password": 0}): + + queued_jobs = get_jobs(site=frappe.local.site, key="job_name")[frappe.local.site] + + for email_account in frappe.get_all( + "Email Account", + filters={"enable_incoming": 1, "awaiting_password": 0}, + fields=["name", "connected_user", "connected_app", "auth_method"], + ): + if email_account.auth_method == "OAuth" and not has_token( + email_account.connected_app, email_account.connected_user + ): + # don't try to pull from accounts which dont have access token (for Oauth) + continue + if now: pull_from_email_account(email_account.name) @@ -966,10 +993,11 @@ def remove_user_email_inbox(email_account): doc.save(ignore_permissions=True) -@frappe.whitelist(allow_guest=False) + +@frappe.whitelist() def set_email_password(email_account, user, password): account = frappe.get_doc("Email Account", email_account) - if account.awaiting_password: + if account.awaiting_password and account.auth_method != "OAuth": account.awaiting_password = 0 account.password = password try: diff --git a/frappe/email/oauth.py b/frappe/email/oauth.py new file mode 100644 index 000000000000..00991e0ea831 --- /dev/null +++ b/frappe/email/oauth.py @@ -0,0 +1,73 @@ +import base64 +from imaplib import IMAP4 +from poplib import POP3 +from smtplib import SMTP + +import frappe + + +class Oauth: + def __init__( + self, + conn, + email_account, + email, + access_token, + mechanism="XOAUTH2", + ): + + self.email_account = email_account + self.email = email + self._mechanism = mechanism + self._conn = conn + self._access_token = access_token + + self._validate() + + def _validate(self) -> None: + if not self._access_token: + frappe.throw( + frappe._("Please Authorize OAuth for Email Account {}").format(self.email_account), + title=frappe._("OAuth Error"), + ) + + @property + def _auth_string(self) -> str: + return f"user={self.email}\1auth=Bearer {self._access_token}\1\1" + + def connect(self) -> None: + try: + if isinstance(self._conn, POP3): + self._connect_pop() + + elif isinstance(self._conn, IMAP4): + self._connect_imap() + + else: + # SMTP + self._connect_smtp() + + except Exception: + frappe.log_error( + title="Email Connection Error - Authentication Failed", + ) + # raising a bare exception here as we have a lot of exception handling present + # where the connect method is called from - hence just logging and raising. + raise + + def _connect_pop(self) -> None: + # NOTE: poplib doesn't have AUTH command implementation + res = self._conn._shortcmd( + "AUTH {} {}".format( + self._mechanism, base64.b64encode(bytes(self._auth_string, "utf-8")).decode("utf-8") + ) + ) + + if not res.startswith(b"+OK"): + raise + + def _connect_imap(self) -> None: + self._conn.authenticate(self._mechanism, lambda x: self._auth_string) + + def _connect_smtp(self) -> None: + self._conn.auth(self._mechanism, lambda x: self._auth_string, initial_response_ok=False) diff --git a/frappe/email/queue.py b/frappe/email/queue.py index 4a6441272134..e445af6ecd66 100755 --- a/frappe/email/queue.py +++ b/frappe/email/queue.py @@ -373,7 +373,10 @@ def flush(from_test=False): msgprint(_("Emails are muted")) from_test = True - smtpserver_dict = frappe._dict() + try: + queued_jobs = set(get_jobs(site=frappe.local.site, key="job_name")[frappe.local.site]) + except Exception: + queued_jobs = set() for email in get_queue(): @@ -381,18 +384,18 @@ def flush(from_test=False): break if email.name: - smtpserver = smtpserver_dict.get(email.sender) - if not smtpserver: - smtpserver = SMTPServer() - smtpserver_dict[email.sender] = smtpserver + job_name = f"email_queue_sendmail_{email.name}" if from_test: - send_one(email.name, smtpserver, auto_commit) + send_one(email.name, auto_commit) else: + if job_name in queued_jobs: + frappe.logger().debug(f"Not queueing job {job_name} because it is in queue already") + continue + send_one_args = { - 'email': email.name, - 'smtpserver': smtpserver, - 'auto_commit': auto_commit, + "email": email.name, + "auto_commit": auto_commit, } enqueue( method = 'frappe.email.queue.send_one', diff --git a/frappe/email/receive.py b/frappe/email/receive.py index 7f254e8884fb..ea0c6673055d 100644 --- a/frappe/email/receive.py +++ b/frappe/email/receive.py @@ -18,6 +18,7 @@ import frappe from frappe import _, safe_decode, safe_encode from frappe.core.doctype.file.file import MaxFileSizeReachedError, get_random_filename +from frappe.email.oauth import Oauth from frappe.utils import ( cint, convert_utc_to_user_timezone, @@ -58,10 +59,7 @@ def process_message(self, mail): def connect(self): """Connect to **Email Account**.""" - if cint(self.settings.use_imap): - return self.connect_imap() - else: - return self.connect_pop() + return self.connect_imap() if cint(self.settings.use_imap) else self.connect_pop() def connect_imap(self): """Connect to IMAP""" @@ -75,7 +73,17 @@ def connect_imap(self): if self.settings.use_starttls: self.imap.starttls() - self.imap.login(self.settings.username, self.settings.password) + if self.settings.use_oauth: + Oauth( + self.imap, + self.settings.email_account, + self.settings.username, + self.settings.access_token, + ).connect() + + else: + self.imap.login(self.settings.username, self.settings.password) + # connection established! return True @@ -92,8 +100,17 @@ def connect_pop(self): else: self.pop = Timed_POP3(self.settings.host, self.settings.incoming_port, timeout=frappe.conf.get("pop_timeout")) - self.pop.user(self.settings.username) - self.pop.pass_(self.settings.password) + if self.settings.use_oauth: + Oauth( + self.pop, + self.settings.email_account, + self.settings.username, + self.settings.access_token, + ).connect() + + else: + self.pop.user(self.settings.username) + self.pop.pass_(self.settings.password) # connection established! return True diff --git a/frappe/email/smtp.py b/frappe/email/smtp.py index 9ba81fa146d0..7ebd1656a85e 100644 --- a/frappe/email/smtp.py +++ b/frappe/email/smtp.py @@ -7,32 +7,9 @@ import email.utils import _socket, sys from frappe import _ +from frappe.email.oauth import Oauth from frappe.utils import cint, cstr, parse_addr -def send(email, append_to=None, retry=1): - """Deprecated: Send the message or add it to Outbox Email""" - def _send(retry): - try: - smtpserver = SMTPServer(append_to=append_to) - - # validate is called in as_string - email_body = email.as_string() - - smtpserver.sess.sendmail(email.sender, email.recipients + (email.cc or []), email_body) - except smtplib.SMTPSenderRefused: - frappe.throw(_("Invalid login or password"), title='Email Failed') - raise - except smtplib.SMTPRecipientsRefused: - frappe.msgprint(_("Invalid recipient address"), title='Email Failed') - raise - except (smtplib.SMTPServerDisconnected, smtplib.SMTPAuthenticationError): - if not retry: - raise - else: - retry = retry - 1 - _send(retry) - - _send(retry) def get_outgoing_email_account(raise_exception_not_set=True, append_to=None, sender=None): """Returns outgoing email account based on `append_to` or the default @@ -60,11 +37,14 @@ def get_outgoing_email_account(raise_exception_not_set=True, append_to=None, sen if not email_account and append_to: # append_to is only valid when enable_incoming is checked - email_accounts = frappe.db.get_values("Email Account", { - "enable_outgoing": 1, - "enable_incoming": 1, - "append_to": append_to, - }, cache=True) + email_accounts = frappe.db.get_values( + "Email Account", + { + "enable_outgoing": 1, + "enable_incoming": 1, + "append_to": append_to, + }, + ) if email_accounts: _email_account = email_accounts[0] @@ -92,7 +72,11 @@ def get_outgoing_email_account(raise_exception_not_set=True, append_to=None, sen if email_account: if email_account.enable_outgoing and not getattr(email_account, 'from_site_config', False): raise_exception = True - if email_account.smtp_server in ['localhost','127.0.0.1'] or email_account.no_smtp_authentication: + if ( + email_account.smtp_server in ["localhost", "127.0.0.1"] + or email_account.no_smtp_authentication + or email_account.auth_method == "OAuth" + ): raise_exception = False email_account.password = email_account.get_password(raise_exception=raise_exception) email_account.default_sender = email.utils.formataddr((email_account.name, email_account.get("email_id"))) @@ -156,7 +140,18 @@ def _get_email_account(filters): return frappe.get_doc("Email Account", name) if name else None class SMTPServer: - def __init__(self, login=None, password=None, server=None, port=None, use_tls=None, use_ssl=None, append_to=None): + def __init__( + self, + login=None, + password=None, + server=None, + port=None, + use_tls=None, + use_ssl=None, + append_to=None, + use_oauth=0, + access_token=None, + ): # get defaults from mail settings self._sess = None @@ -171,7 +166,8 @@ def __init__(self, login=None, password=None, server=None, port=None, use_tls=No self.use_ssl = cint(use_ssl) self.login = login self.password = password - + self.use_oauth = use_oauth + self.access_token = access_token else: self.setup_email_account(append_to) @@ -195,6 +191,10 @@ def setup_email_account(self, append_to=None, sender=None): self.always_use_account_email_id_as_sender = cint(self.email_account.get("always_use_account_email_id_as_sender")) self.always_use_account_name_as_sender_name = cint(self.email_account.get("always_use_account_name_as_sender_name")) + oauth_token = self.email_account.get_oauth_token() + self.use_oauth = self.email_account.auth_method == "OAuth" + self.access_token = oauth_token.get_password("access_token") if oauth_token else None + @property def sess(self): """get session""" @@ -230,7 +230,10 @@ def sess(self): self._sess.starttls() self._sess.ehlo() - if self.login and self.password: + if self.use_oauth: + Oauth(self._sess, self.email_account, self.login, self.access_token).connect() + + elif self.password: ret = self._sess.login(str(self.login or ""), str(self.password or "")) # check if logged correctly diff --git a/frappe/integrations/doctype/connected_app/connected_app.py b/frappe/integrations/doctype/connected_app/connected_app.py index 449e30f6d09f..dd9101620645 100644 --- a/frappe/integrations/doctype/connected_app/connected_app.py +++ b/frappe/integrations/doctype/connected_app/connected_app.py @@ -13,7 +13,10 @@ if any((os.getenv('CI'), frappe.conf.developer_mode, frappe.conf.allow_tests)): # Disable mandatory TLS in developer mode and tests - os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = '1' + os.environ["OAUTHLIB_INSECURE_TRANSPORT"] = "1" + +os.environ["OAUTHLIB_RELAX_TOKEN_SCOPE"] = "1" + class ConnectedApp(Document): """Connect to a remote oAuth Server. Retrieve and store user's access token @@ -55,7 +58,7 @@ def get_oauth2_session(self, user=None, init=False): def initiate_web_application_flow(self, user=None, success_uri=None): """Return an authorization URL for the user. Save state in Token Cache.""" user = user or frappe.session.user - oauth = self.get_oauth2_session(init=True) + oauth = self.get_oauth2_session(user, init=True) query_params = self.get_query_params() authorization_url, state = oauth.authorization_url(self.authorization_uri, **query_params) token_cache = self.get_token_cache(user) @@ -94,6 +97,25 @@ def get_token_cache(self, user): return token_cache + def get_active_token(self, user=None): + user = user or frappe.session.user + token_cache = self.get_token_cache(user) + if token_cache and token_cache.is_expired(): + oauth_session = self.get_oauth2_session(user) + + try: + token = oauth_session.refresh_token( + body=f"redirect_uri={self.redirect_uri}", + token_url=self.token_uri, + ) + except Exception: + frappe.log_error(title="Token Refresh Error") + return None + + token_cache.update_data(token) + + return token_cache + def get_scopes(self): return [row.scope for row in self.scopes] @@ -101,7 +123,7 @@ def get_query_params(self): return {param.key: param.value for param in self.query_parameters} -@frappe.whitelist(allow_guest=True) +@frappe.whitelist(methods=["GET"], allow_guest=True) def callback(code=None, state=None): """Handle client's code. @@ -109,8 +131,6 @@ def callback(code=None, state=None): transmit a code that can be used by the local server to obtain an access token. """ - if frappe.request.method != 'GET': - frappe.throw(_('Invalid request method: {}').format(frappe.request.method)) if frappe.session.user == 'Guest': frappe.local.response['type'] = 'redirect' @@ -133,9 +153,16 @@ def callback(code=None, state=None): code=code, client_secret=connected_app.get_password('client_secret'), include_client_id=True, - **query_params + **query_params, ) token_cache.update_data(token) - frappe.local.response['type'] = 'redirect' - frappe.local.response['location'] = token_cache.get('success_uri') or connected_app.get_url() + frappe.local.response["type"] = "redirect" + frappe.local.response["location"] = token_cache.get("success_uri") or connected_app.get_url() + + +@frappe.whitelist() +def has_token(connected_app, connected_user=None): + app = frappe.get_doc("Connected App", connected_app) + token_cache = app.get_token_cache(connected_user or frappe.session.user) + return bool(token_cache and token_cache.get_password("access_token", False)) diff --git a/frappe/integrations/doctype/token_cache/token_cache.json b/frappe/integrations/doctype/token_cache/token_cache.json index 3dab457a834d..8f3ddad54004 100644 --- a/frappe/integrations/doctype/token_cache/token_cache.json +++ b/frappe/integrations/doctype/token_cache/token_cache.json @@ -86,7 +86,7 @@ } ], "links": [], - "modified": "2021-05-12 19:59:44.251235", + "modified": "2021-05-13 19:59:44.251235", "modified_by": "Administrator", "module": "Integrations", "name": "Token Cache", @@ -106,6 +106,5 @@ ], "restrict_to_domain": "Integrations", "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1 + "sort_order": "DESC" } \ No newline at end of file diff --git a/frappe/integrations/doctype/token_cache/token_cache.py b/frappe/integrations/doctype/token_cache/token_cache.py index 3e87a2e44902..658f3d3e32c3 100644 --- a/frappe/integrations/doctype/token_cache/token_cache.py +++ b/frappe/integrations/doctype/token_cache/token_cache.py @@ -3,12 +3,17 @@ # For license information, please see license.txt from __future__ import unicode_literals -from datetime import timedelta + +from datetime import datetime, timedelta + +import pytz import frappe from frappe import _ from frappe.utils import cstr, cint from frappe.model.document import Document +from frappe.utils import cint, cstr, get_time_zone + class TokenCache(Document): @@ -52,16 +57,19 @@ def update_data(self, data): return self def get_expires_in(self): - expiry_time = frappe.utils.get_datetime(self.modified) + timedelta(seconds=self.expires_in) - return (expiry_time - frappe.utils.now_datetime()).total_seconds() + system_timezone = pytz.timezone(get_time_zone()) + modified = system_timezone.localize(frappe.utils.get_datetime(self.modified)) + expiry_utc = modified.astimezone(pytz.utc) + timedelta(seconds=self.expires_in) + now_utc = datetime.utcnow().replace(tzinfo=pytz.utc) + return cint((expiry_utc - now_utc).total_seconds()) def is_expired(self): return self.get_expires_in() < 0 def get_json(self): return { - 'access_token': self.get_password('access_token', ''), - 'refresh_token': self.get_password('refresh_token', ''), - 'expires_in': self.get_expires_in(), - 'token_type': self.token_type + "access_token": self.get_password("access_token", False), + "refresh_token": self.get_password("refresh_token", False), + "expires_in": self.get_expires_in(), + "token_type": self.token_type, } From bbdf3342dc5bf26ca96f020ade7427a543e3f48d Mon Sep 17 00:00:00 2001 From: fproldan Date: Mon, 21 Oct 2024 15:24:40 +0000 Subject: [PATCH 07/20] feat: oauth --- frappe/core/doctype/communication/email.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/core/doctype/communication/email.py b/frappe/core/doctype/communication/email.py index ec7c04d3c5f5..0ef491ceb88b 100755 --- a/frappe/core/doctype/communication/email.py +++ b/frappe/core/doctype/communication/email.py @@ -28,7 +28,7 @@ def make(doctype=None, name=None, content=None, subject=None, sent_or_received = "Sent", sender=None, sender_full_name=None, recipients=None, communication_medium="Email", send_email=False, print_html=None, print_format=None, attachments='[]', send_me_a_copy=False, cc=None, bcc=None, - flags=None, read_receipt=None, print_letterhead=True, email_template=None, communication_type=None, + flags=None, read_receipt=None, print_letterhead=True, email_template=None, communication_type=None, add_signature=True, ignore_permissions=False): """Make a new communication. From 447cf5787dc2830eda181ce0ffd00bf797f528c6 Mon Sep 17 00:00:00 2001 From: leela Date: Wed, 22 Sep 2021 11:29:01 +0530 Subject: [PATCH 08/20] fix: Raise email account missing error if found while sending mail From b66913764f856b97ccaa0d6aaddf1a68b97d0a42 Mon Sep 17 00:00:00 2001 From: leela Date: Wed, 22 Sep 2021 11:38:28 +0530 Subject: [PATCH 09/20] fix: Sendmail is failing randomly This is happening because we enqueue the sendmail task before the current transaction is commited. enqueued task is looking for a document that is not yet commited and thus failing to send mails. From ceefcf3f3f95dd57a9d6539a6c51ca3b062d4c6d Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Wed, 17 Nov 2021 14:47:41 +0530 Subject: [PATCH 10/20] fix: optimise `mark_email_as_seen` (cherry picked from commit 69c87e5caec236ed2b09f52dec73c5c6a2426f9d) --- frappe/core/doctype/communication/email.py | 54 +++++++++++++++------- 1 file changed, 37 insertions(+), 17 deletions(-) diff --git a/frappe/core/doctype/communication/email.py b/frappe/core/doctype/communication/email.py index 0ef491ceb88b..e78effa03ba2 100755 --- a/frappe/core/doctype/communication/email.py +++ b/frappe/core/doctype/communication/email.py @@ -487,24 +487,44 @@ def sendmail(communication_name, print_html=None, print_format=None, attachments raise @frappe.whitelist(allow_guest=True) -def mark_email_as_seen(name=None): +def mark_email_as_seen(name: str = None): try: - if name and frappe.db.exists("Communication", name) and not frappe.db.get_value("Communication", name, "read_by_recipient"): - frappe.db.set_value("Communication", name, "read_by_recipient", 1) - frappe.db.set_value("Communication", name, "delivery_status", "Read") - frappe.db.set_value("Communication", name, "read_by_recipient_on", get_datetime()) - frappe.db.commit() + update_communication_as_seen(name) + except Exception: frappe.log_error(frappe.get_traceback()) + finally: - # Return image as response under all circumstances - from PIL import Image - import io - im = Image.new('RGBA', (1, 1)) - im.putdata([(255,255,255,0)]) - buffered_obj = io.BytesIO() - im.save(buffered_obj, format="PNG") - - frappe.response["type"] = 'binary' - frappe.response["filename"] = "imaginary_pixel.png" - frappe.response["filecontent"] = buffered_obj.getvalue() + frappe.response.update({ + "type": "binary", + "filename": "imaginary_pixel.png", + "filecontent": ( + b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00" + b"\x00\x01\x08\x06\x00\x00\x00\x1f\x15\xc4\x89\x00\x00\x00\r" + b"IDATx\x9cc\xf8\xff\xff?\x03\x00\x08\xfc\x02\xfe\xa7\x9a\xa0" + b"\xa0\x00\x00\x00\x00IEND\xaeB`\x82" + ) + }) + +def update_communication_as_seen(name): + if not name or not isinstance(name, str): + return + + values = frappe.db.get_value( + "Communication", + name, + "read_by_recipient", + as_dict=True + ) + + # Communication not found or already marked read + if not values or values.read_by_recipient: + return + + frappe.db.set_value("Communication", name, { + "read_by_recipient": 1, + "delivery_status": "Read", + "read_by_recipient_on": get_datetime() + }) + + frappe.db.commit() From 0804895174cb04844c15c8d4beaa1d11bb790262 Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Wed, 17 Nov 2021 14:55:20 +0530 Subject: [PATCH 11/20] fix: enforce GET method (cherry picked from commit a80cf47426947651315151cc989211eae7ba23bf) # Conflicts: # frappe/core/doctype/communication/email.py --- frappe/core/doctype/communication/email.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frappe/core/doctype/communication/email.py b/frappe/core/doctype/communication/email.py index e78effa03ba2..ed2abcb676f2 100755 --- a/frappe/core/doctype/communication/email.py +++ b/frappe/core/doctype/communication/email.py @@ -389,6 +389,7 @@ def add_attachments(name, attachments): }) _file.save(ignore_permissions=True) +<<<<<<< HEAD def filter_email_list(doc, email_list, exclude, is_cc=False, is_bcc=False): # temp variables filtered = [] @@ -487,6 +488,9 @@ def sendmail(communication_name, print_html=None, print_format=None, attachments raise @frappe.whitelist(allow_guest=True) +======= +@frappe.whitelist(allow_guest=True, methods=("GET",)) +>>>>>>> a80cf47426 (fix: enforce GET method) def mark_email_as_seen(name: str = None): try: update_communication_as_seen(name) From 96ec563b02eb468736c14ac93efa5fce52525600 Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Wed, 17 Nov 2021 15:44:49 +0530 Subject: [PATCH 12/20] fix: move commit call to `mark_email_as_seen` (cherry picked from commit 7b7f74ce23c2d9af7d5d6ee01e209385778aa51c) --- frappe/core/doctype/communication/email.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/frappe/core/doctype/communication/email.py b/frappe/core/doctype/communication/email.py index ed2abcb676f2..392b63d13bca 100755 --- a/frappe/core/doctype/communication/email.py +++ b/frappe/core/doctype/communication/email.py @@ -494,6 +494,7 @@ def sendmail(communication_name, print_html=None, print_format=None, attachments def mark_email_as_seen(name: str = None): try: update_communication_as_seen(name) + frappe.db.commit() # nosemgrep: this will be called in a GET request except Exception: frappe.log_error(frappe.get_traceback()) @@ -530,5 +531,3 @@ def update_communication_as_seen(name): "delivery_status": "Read", "read_by_recipient_on": get_datetime() }) - - frappe.db.commit() From 91f9ca366fb2e31cc0ab55a518afa93dc996ee46 Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Wed, 17 Nov 2021 16:34:47 +0530 Subject: [PATCH 13/20] fix: slightly better naming (cherry picked from commit cb97b49c78ef9df61eb683d09d262651a738b392) --- frappe/core/doctype/communication/email.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/frappe/core/doctype/communication/email.py b/frappe/core/doctype/communication/email.py index 392b63d13bca..cb066297cd16 100755 --- a/frappe/core/doctype/communication/email.py +++ b/frappe/core/doctype/communication/email.py @@ -493,7 +493,7 @@ def sendmail(communication_name, print_html=None, print_format=None, attachments >>>>>>> a80cf47426 (fix: enforce GET method) def mark_email_as_seen(name: str = None): try: - update_communication_as_seen(name) + update_communication_as_read(name) frappe.db.commit() # nosemgrep: this will be called in a GET request except Exception: @@ -511,19 +511,18 @@ def mark_email_as_seen(name: str = None): ) }) -def update_communication_as_seen(name): +def update_communication_as_read(name): if not name or not isinstance(name, str): return - values = frappe.db.get_value( + communication = frappe.db.get_value( "Communication", name, "read_by_recipient", as_dict=True ) - # Communication not found or already marked read - if not values or values.read_by_recipient: + if not communication or communication.read_by_recipient: return frappe.db.set_value("Communication", name, { From bf580d1cafe8265b5612609a5b160dae0273fee6 Mon Sep 17 00:00:00 2001 From: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> Date: Mon, 22 Nov 2021 11:00:03 +0530 Subject: [PATCH 14/20] fix: Resolve conflicts --- frappe/core/doctype/communication/email.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/frappe/core/doctype/communication/email.py b/frappe/core/doctype/communication/email.py index cb066297cd16..e72ec3fa5c47 100755 --- a/frappe/core/doctype/communication/email.py +++ b/frappe/core/doctype/communication/email.py @@ -389,7 +389,6 @@ def add_attachments(name, attachments): }) _file.save(ignore_permissions=True) -<<<<<<< HEAD def filter_email_list(doc, email_list, exclude, is_cc=False, is_bcc=False): # temp variables filtered = [] @@ -487,10 +486,7 @@ def sendmail(communication_name, print_html=None, print_format=None, attachments traceback = frappe.log_error("frappe.core.doctype.communication.email.sendmail") raise -@frappe.whitelist(allow_guest=True) -======= @frappe.whitelist(allow_guest=True, methods=("GET",)) ->>>>>>> a80cf47426 (fix: enforce GET method) def mark_email_as_seen(name: str = None): try: update_communication_as_read(name) From 544f5a0126b5223e2a2025a53e3c54b5c670ceb3 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Wed, 9 Mar 2022 11:39:11 +0000 Subject: [PATCH 15/20] fix: Double signature in composed Email (backport #16178) (#16228) This is a semi-automatic backport of pull request #16178 done by [Mergify](https://mergify.com). --- .../doctype/communication/communication.py | 39 ++++++++ frappe/core/doctype/communication/email.py | 98 +++++++++++++++++-- .../doctype/notification/notification.py | 7 +- frappe/email/email_body.py | 8 +- .../public/js/frappe/views/communication.js | 2 +- frappe/templates/emails/standard.html | 1 - requirements.txt | 1 + 7 files changed, 135 insertions(+), 21 deletions(-) diff --git a/frappe/core/doctype/communication/communication.py b/frappe/core/doctype/communication/communication.py index 5ebf71464576..a2d6fefb8d00 100644 --- a/frappe/core/doctype/communication/communication.py +++ b/frappe/core/doctype/communication/communication.py @@ -17,6 +17,7 @@ from frappe.utils.user import is_system_user from frappe.contacts.doctype.contact.contact import get_contact_name from frappe.automation.doctype.assignment_rule.assignment_rule import apply as apply_assignment_rule +from parse import compile exclude_from_linked_with = True @@ -111,6 +112,44 @@ def after_insert(self): frappe.publish_realtime('new_message', self.as_dict(), user=self.reference_name, after_commit=True) + def set_signature_in_email_content(self): + """Set sender's User.email_signature or default outgoing's EmailAccount.signature to the email + """ + if not self.content: + return + + quill_parser = compile('
{}
') + email_body = quill_parser.parse(self.content) + + if not email_body: + return + + email_body = email_body[0] + + user_email_signature = frappe.db.get_value( + "User", + self.sender, + "email_signature", + ) if self.sender else None + + signature = user_email_signature or frappe.db.get_value( + "Email Account", + {"default_outgoing": 1, "add_signature": 1}, + "signature", + ) + + if not signature: + return + + _signature = quill_parser.parse(signature)[0] if "ql-editor" in signature else None + + if (_signature or signature) not in self.content: + self.content = f'{self.content}


{signature}' + + def before_save(self): + if not self.flags.skip_add_signature: + self.set_signature_in_email_content() + def on_update(self): # add to _comment property of the doctype, so it shows up in # comments count for the list view diff --git a/frappe/core/doctype/communication/email.py b/frappe/core/doctype/communication/email.py index e72ec3fa5c47..1ad03bea84a4 100755 --- a/frappe/core/doctype/communication/email.py +++ b/frappe/core/doctype/communication/email.py @@ -28,9 +28,14 @@ def make(doctype=None, name=None, content=None, subject=None, sent_or_received = "Sent", sender=None, sender_full_name=None, recipients=None, communication_medium="Email", send_email=False, print_html=None, print_format=None, attachments='[]', send_me_a_copy=False, cc=None, bcc=None, +<<<<<<< HEAD flags=None, read_receipt=None, print_letterhead=True, email_template=None, communication_type=None, add_signature=True, ignore_permissions=False): """Make a new communication. +======= + read_receipt=None, print_letterhead=True, email_template=None, communication_type=None, **kwargs): + """Make a new communication. Checks for email permissions for specified Document. +>>>>>>> 852ce50445 (fix: Double signature in composed Email (backport #16178) (#16228)) :param doctype: Reference DocType. :param name: Reference Document name. @@ -47,22 +52,76 @@ def make(doctype=None, name=None, content=None, subject=None, sent_or_received = :param send_me_a_copy: Send a copy to the sender (default **False**). :param email_template: Template which is used to compose mail . """ + if kwargs: + from frappe.utils.commands import warn + warn( + f"Options {kwargs} used in frappe.core.doctype.communication.email.make " + "are deprecated or unsupported", + category=DeprecationWarning + ) + + if doctype and name and not frappe.has_permission(doctype=doctype, ptype="email", doc=name): + raise frappe.PermissionError( + f"You are not allowed to send emails related to: {doctype} {name}" + ) + + return _make( + doctype=doctype, + name=name, + content=content, + subject=subject, + sent_or_received=sent_or_received, + sender=sender, + sender_full_name=sender_full_name, + recipients=recipients, + communication_medium=communication_medium, + send_email=send_email, + print_html=print_html, + print_format=print_format, + attachments=attachments, + send_me_a_copy=cint(send_me_a_copy), + cc=cc, + bcc=bcc, + read_receipt=read_receipt, + print_letterhead=print_letterhead, + email_template=email_template, + communication_type=communication_type, + add_signature=False, + ) - is_error_report = (doctype=="User" and name==frappe.session.user and subject=="Error Report") - send_me_a_copy = cint(send_me_a_copy) - - if not ignore_permissions: - if doctype and name and not is_error_report and not frappe.has_permission(doctype, "email", name) and not (flags or {}).get('ignore_doctype_permissions'): - raise frappe.PermissionError("You are not allowed to send emails related to: {doctype} {name}".format( - doctype=doctype, name=name)) - if not sender: - sender = get_formatted_email(frappe.session.user) +def _make( + doctype=None, + name=None, + content=None, + subject=None, + sent_or_received="Sent", + sender=None, + sender_full_name=None, + recipients=None, + communication_medium="Email", + send_email=False, + print_html=None, + print_format=None, + attachments="[]", + send_me_a_copy=False, + cc=None, + bcc=None, + read_receipt=None, + print_letterhead=True, + email_template=None, + communication_type=None, + add_signature=True, +): + """Internal method to make a new communication that ignores Permission checks. + """ + sender = sender or get_formatted_email(frappe.session.user) recipients = list_to_str(recipients) if isinstance(recipients, list) else recipients cc = list_to_str(cc) if isinstance(cc, list) else cc bcc = list_to_str(bcc) if isinstance(bcc, list) else bcc +<<<<<<< HEAD comm = frappe.get_doc( { "doctype": "Communication", @@ -84,6 +143,27 @@ def make(doctype=None, name=None, content=None, subject=None, sent_or_received = "communication_type": communication_type, } ) +======= + comm = frappe.get_doc({ + "doctype":"Communication", + "subject": subject, + "content": content, + "sender": sender, + "sender_full_name":sender_full_name, + "recipients": recipients, + "cc": cc or None, + "bcc": bcc or None, + "communication_medium": communication_medium, + "sent_or_received": sent_or_received, + "reference_doctype": doctype, + "reference_name": name, + "email_template": email_template, + "message_id":get_message_id().strip(" <>"), + "read_receipt":read_receipt, + "has_attachment": 1 if attachments else 0, + "communication_type": communication_type, + }) +>>>>>>> 852ce50445 (fix: Double signature in composed Email (backport #16178) (#16228)) comm.flags.skip_add_signature = not add_signature comm.insert(ignore_permissions=True) diff --git a/frappe/email/doctype/notification/notification.py b/frappe/email/doctype/notification/notification.py index 6130fb2e570e..ed7b28f53e21 100644 --- a/frappe/email/doctype/notification/notification.py +++ b/frappe/email/doctype/notification/notification.py @@ -190,7 +190,7 @@ def create_system_notification(self, doc, context): def send_an_email(self, doc, context): from email.utils import formataddr - from frappe.core.doctype.communication.email import make as make_communication + from frappe.core.doctype.communication.email import _make as make_communication subject = self.subject if "{" in subject: subject = frappe.render_template(self.subject, context) @@ -220,7 +220,8 @@ def send_an_email(self, doc, context): # Add mail notification to communication list # No need to add if it is already a communication. if doc.doctype != 'Communication': - make_communication(doctype=doc.doctype, + make_communication( + doctype=doc.doctype, name=doc.name, content=message, subject=subject, @@ -232,7 +233,7 @@ def send_an_email(self, doc, context): cc=cc, bcc=bcc, communication_type='Automated Message', - ignore_permissions=True) + ) def send_a_slack_msg(self, doc, context): send_slack_message( diff --git a/frappe/email/email_body.py b/frappe/email/email_body.py index 3dcdf00a8e3b..30b5f100f10b 100755 --- a/frappe/email/email_body.py +++ b/frappe/email/email_body.py @@ -252,17 +252,12 @@ def get_formatted_html(subject, message, footer=None, print_html=None, if not email_account: email_account = get_outgoing_email_account(False, sender=sender) - signature = None - if "" not in message: - signature = get_signature(email_account) - rendered_email = frappe.get_template("templates/emails/standard.html").render({ "brand_logo": get_brand_logo(email_account) if with_container or header else None, "with_container": with_container, "site_url": get_url(), "header": get_header(header), "content": message, - "signature": signature, "footer": get_footer(email_account, footer), "title": subject, "print_html": print_html, @@ -274,8 +269,7 @@ def get_formatted_html(subject, message, footer=None, print_html=None, if unsubscribe_link: html = html.replace("", unsubscribe_link.html) - html = inline_style_in_html(html) - return html + return inline_style_in_html(html) @frappe.whitelist() def get_email_html(template, args, subject, header=None, with_container=False): diff --git a/frappe/public/js/frappe/views/communication.js b/frappe/public/js/frappe/views/communication.js index b33cd1a3f181..0ac8c59d6f9b 100755 --- a/frappe/public/js/frappe/views/communication.js +++ b/frappe/public/js/frappe/views/communication.js @@ -758,7 +758,7 @@ frappe.views.CommunicationComposer = class { signature = signature.replace(/\n/g, "
"); } - return "
" + signature; + return "
" + signature; } get_earlier_reply() { diff --git a/frappe/templates/emails/standard.html b/frappe/templates/emails/standard.html index 4a47c9cf9071..2a2093e1e920 100644 --- a/frappe/templates/emails/standard.html +++ b/frappe/templates/emails/standard.html @@ -37,7 +37,6 @@

{{ content }}

-

{{ signature }}

diff --git a/requirements.txt b/requirements.txt index a011d5a2f3aa..258874ef6338 100644 --- a/requirements.txt +++ b/requirements.txt @@ -35,6 +35,7 @@ ndg-httpsclient~=0.5.1 num2words~=0.5.10 oauthlib~=3.1.0 openpyxl~=3.0.7 +parse~=1.19.0 passlib~=1.7.4 paytmchecksum~=1.7.0 pdfkit~=0.6.1 From 3d40152cf5b549ee0155a35b20abdbd3cee51f1b Mon Sep 17 00:00:00 2001 From: Shariq Ansari <30859809+shariquerik@users.noreply.github.com> Date: Fri, 27 May 2022 21:06:20 +0530 Subject: [PATCH 16/20] fix: Strip all spacing characters from Message-ID & In-Reply-To (backport #16999) (#17004) --- frappe/utils/data.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 4cfdbfd2540d..d8bc91824b41 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -1468,6 +1468,15 @@ def get_string_between(start, string, end): return out.group(1) if out else string +def to_markdown(html): + from html.parser import HTMLParser + + regex = "{0}(.*){1}".format(start, end) + out = re.search(regex, string) + + return out.group(1) if out else string + + def to_markdown(html): from html2text import html2text from six.moves import html_parser as HTMLParser From af2d6f72801385605b0a7dca75e8c193ccab52b4 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Tue, 31 May 2022 13:47:18 +0530 Subject: [PATCH 17/20] feat: Auto-expire web view link key --- frappe/core/doctype/communication/email.py | 46 +++--------- .../doctype/document_share_key/__init__.py | 0 .../document_share_key/document_share_key.js | 8 +++ .../document_share_key.json | 71 +++++++++++++++++++ .../document_share_key/document_share_key.py | 17 +++++ .../system_settings/system_settings.json | 40 ++++++++++- .../desk/doctype/kanban_board/kanban_board.py | 24 ++++++- frappe/exceptions.py | 31 ++++++-- frappe/model/document.py | 24 +++++++ frappe/public/scss/print.scss | 37 ++++++++-- .../print_formats/print_key_expired.html | 11 +++ .../print_formats/print_key_invalid.html | 14 ++++ frappe/utils/print_format.py | 19 ++++- frappe/www/printview.html | 29 ++++++-- frappe/www/printview.py | 68 ++++++++++++++---- 15 files changed, 364 insertions(+), 75 deletions(-) create mode 100644 frappe/core/doctype/document_share_key/__init__.py create mode 100644 frappe/core/doctype/document_share_key/document_share_key.js create mode 100644 frappe/core/doctype/document_share_key/document_share_key.json create mode 100644 frappe/core/doctype/document_share_key/document_share_key.py create mode 100644 frappe/templates/print_formats/print_key_expired.html create mode 100644 frappe/templates/print_formats/print_key_invalid.html diff --git a/frappe/core/doctype/communication/email.py b/frappe/core/doctype/communication/email.py index 1ad03bea84a4..d3743e83cc6a 100755 --- a/frappe/core/doctype/communication/email.py +++ b/frappe/core/doctype/communication/email.py @@ -28,14 +28,8 @@ def make(doctype=None, name=None, content=None, subject=None, sent_or_received = "Sent", sender=None, sender_full_name=None, recipients=None, communication_medium="Email", send_email=False, print_html=None, print_format=None, attachments='[]', send_me_a_copy=False, cc=None, bcc=None, -<<<<<<< HEAD - flags=None, read_receipt=None, print_letterhead=True, email_template=None, communication_type=None, add_signature=True, - ignore_permissions=False): - """Make a new communication. -======= read_receipt=None, print_letterhead=True, email_template=None, communication_type=None, **kwargs): """Make a new communication. Checks for email permissions for specified Document. ->>>>>>> 852ce50445 (fix: Double signature in composed Email (backport #16178) (#16228)) :param doctype: Reference DocType. :param name: Reference Document name. @@ -121,29 +115,6 @@ def _make( cc = list_to_str(cc) if isinstance(cc, list) else cc bcc = list_to_str(bcc) if isinstance(bcc, list) else bcc -<<<<<<< HEAD - comm = frappe.get_doc( - { - "doctype": "Communication", - "subject": subject, - "content": content, - "sender": sender, - "sender_full_name": sender_full_name, - "recipients": recipients, - "cc": cc or None, - "bcc": bcc or None, - "communication_medium": communication_medium, - "sent_or_received": sent_or_received, - "reference_doctype": doctype, - "reference_name": name, - "email_template": email_template, - "message_id": get_string_between("<", get_message_id(), ">"), - "read_receipt": read_receipt, - "has_attachment": 1 if attachments else 0, - "communication_type": communication_type, - } - ) -======= comm = frappe.get_doc({ "doctype":"Communication", "subject": subject, @@ -163,7 +134,6 @@ def _make( "has_attachment": 1 if attachments else 0, "communication_type": communication_type, }) ->>>>>>> 852ce50445 (fix: Double signature in composed Email (backport #16178) (#16228)) comm.flags.skip_add_signature = not add_signature comm.insert(ignore_permissions=True) @@ -521,13 +491,15 @@ def get_assignees(doc): def get_attach_link(doc, print_format): """Returns public link for the attachment via `templates/emails/print_link.html`.""" - return frappe.get_template("templates/emails/print_link.html").render({ - "url": get_url(), - "doctype": doc.reference_doctype, - "name": doc.reference_name, - "print_format": print_format, - "key": get_parent_doc(doc).get_signature() - }) + return frappe.get_template("templates/emails/print_link.html").render( + { + "url": get_url(), + "doctype": doc.reference_doctype, + "name": doc.reference_name, + "print_format": print_format, + "key": get_parent_doc(doc).get_document_share_key(), + } + ) def sendmail(communication_name, print_html=None, print_format=None, attachments=None, recipients=None, cc=None, bcc=None, lang=None, session=None, print_letterhead=None): diff --git a/frappe/core/doctype/document_share_key/__init__.py b/frappe/core/doctype/document_share_key/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/frappe/core/doctype/document_share_key/document_share_key.js b/frappe/core/doctype/document_share_key/document_share_key.js new file mode 100644 index 000000000000..1ebf0de4ce6f --- /dev/null +++ b/frappe/core/doctype/document_share_key/document_share_key.js @@ -0,0 +1,8 @@ +// Copyright (c) 2022, Frappe Technologies and contributors +// For license information, please see license.txt + +frappe.ui.form.on('Document Share Key', { + // refresh: function(frm) { + + // } +}); diff --git a/frappe/core/doctype/document_share_key/document_share_key.json b/frappe/core/doctype/document_share_key/document_share_key.json new file mode 100644 index 000000000000..ff6eed507661 --- /dev/null +++ b/frappe/core/doctype/document_share_key/document_share_key.json @@ -0,0 +1,71 @@ +{ + "actions": [], + "allow_rename": 1, + "autoname": "hash", + "creation": "2022-05-31 08:11:32.357683", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "reference_doctype", + "reference_docname", + "key", + "expires_on" + ], + "fields": [ + { + "fieldname": "reference_doctype", + "fieldtype": "Link", + "label": "Reference Document Type", + "options": "DocType", + "read_only": 1, + "search_index": 1 + }, + { + "fieldname": "reference_docname", + "fieldtype": "Dynamic Link", + "label": "Reference Document Name", + "options": "reference_doctype", + "read_only": 1, + "search_index": 1 + }, + { + "fieldname": "key", + "fieldtype": "Data", + "label": "Key", + "read_only": 1 + }, + { + "fieldname": "expires_on", + "fieldtype": "Date", + "in_list_view": 1, + "label": "Expires On", + "read_only": 1 + } + ], + "in_create": 1, + "index_web_pages_for_search": 1, + "links": [], + "modified": "2022-05-31 08:11:32.357683", + "modified_by": "Administrator", + "module": "Core", + "name": "Document Share Key", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, + "write": 1 + } + ], + "read_only": 1, + "sort_field": "modified", + "sort_order": "DESC" +} \ No newline at end of file diff --git a/frappe/core/doctype/document_share_key/document_share_key.py b/frappe/core/doctype/document_share_key/document_share_key.py new file mode 100644 index 000000000000..dc979ee6375e --- /dev/null +++ b/frappe/core/doctype/document_share_key/document_share_key.py @@ -0,0 +1,17 @@ +# Copyright (c) 2022, Frappe Technologies and contributors +# For license information, please see license.txt + +import frappe +from frappe.model.document import Document + + +class DocumentShareKey(Document): + def before_insert(self): + self.key = frappe.generate_hash(length=32) + if not self.expires_on and not self.flags.no_expiry: + self.expires_on = frappe.utils.add_days( + None, days=frappe.get_system_settings("document_share_key_expiry") or 90 + ) + + def is_expired(self): + return self.expires_on and self.expires_on < frappe.utils.getdate() diff --git a/frappe/core/doctype/system_settings/system_settings.json b/frappe/core/doctype/system_settings/system_settings.json index bd923315af99..a505807a690a 100644 --- a/frappe/core/doctype/system_settings/system_settings.json +++ b/frappe/core/doctype/system_settings/system_settings.json @@ -33,12 +33,14 @@ "security", "session_expiry", "session_expiry_mobile", + "document_share_key_expiry", "column_break_13", "deny_multiple_sessions", "allow_login_using_mobile_number", "allow_login_using_user_name", "allow_error_traceback", "strip_exif_metadata_from_uploaded_images", + "allow_older_web_view_links", "password_settings", "logout_on_password_reset", "force_user_to_reset_password", @@ -507,12 +509,48 @@ "fieldtype": "Check", "hidden": 1, "label": "Disable System Update Notification" + }, + { + "default": "Sunday", + "fieldname": "first_day_of_the_week", + "fieldtype": "Select", + "label": "First Day of the Week", + "options": "Sunday\nMonday\nTuesday\nWednesday\nThursday\nFriday\nSaturday" + }, + { + "fieldname": "column_break_64", + "fieldtype": "Column Break" + }, + { + "default": "20", + "fieldname": "max_auto_email_report_per_user", + "fieldtype": "Int", + "label": "Max auto email report per user" + }, + { + "default": "0", + "fieldname": "disable_change_log_notification", + "fieldtype": "Check", + "label": "Disable Change Log Notification" + }, + { + "default": "30", + "description": "Number of days after which the document Web View link shared on email will be expired", + "fieldname": "document_share_key_expiry", + "fieldtype": "Int", + "label": "Document Share Key Expiry (in Days)" + }, + { + "default": "0", + "fieldname": "allow_older_web_view_links", + "fieldtype": "Check", + "label": "Allow Older Web View Links (Insecure)" } ], "icon": "fa fa-cog", "issingle": 1, "links": [], - "modified": "2022-08-16 12:32:23.736407", + "modified": "2022-08-16 13:32:23.736407", "modified_by": "Administrator", "module": "Core", "name": "System Settings", diff --git a/frappe/desk/doctype/kanban_board/kanban_board.py b/frappe/desk/doctype/kanban_board/kanban_board.py index a655e9e1da2d..93e51a73e6d5 100644 --- a/frappe/desk/doctype/kanban_board/kanban_board.py +++ b/frappe/desk/doctype/kanban_board/kanban_board.py @@ -264,6 +264,24 @@ def set_indicator(board_name, column_name, indicator): @frappe.whitelist() def save_filters(board_name, filters): - '''Save filters silently''' - frappe.db.set_value('Kanban Board', board_name, 'filters', - filters, update_modified=False) + """Save filters silently""" + frappe.db.set_value("Kanban Board", board_name, "filters", filters, update_modified=False) + + +@frappe.whitelist() +def save_settings(board_name: str, settings: str) -> Document: + settings = json.loads(settings) + doc = frappe.get_doc("Kanban Board", board_name) + + fields = settings["fields"] + if not isinstance(fields, str): + fields = json.dumps(fields) + + doc.fields = fields + doc.show_labels = settings["show_labels"] + doc.save() + + resp = doc.as_dict() + resp["fields"] = frappe.parse_json(resp["fields"]) + + return resp diff --git a/frappe/exceptions.py b/frappe/exceptions.py index 4dd1bef61fd6..b8e757c85402 100644 --- a/frappe/exceptions.py +++ b/frappe/exceptions.py @@ -113,8 +113,29 @@ class AttachmentLimitReached(ValidationError): pass class QueryTimeoutError(ValidationError): pass class QueryDeadlockError(ValidationError): pass # OAuth exceptions -class InvalidAuthorizationHeader(CSRFTokenError): pass -class InvalidAuthorizationPrefix(CSRFTokenError): pass -class InvalidAuthorizationToken(CSRFTokenError): pass -class InvalidDatabaseFile(ValidationError): pass -class ExecutableNotFound(FileNotFoundError): pass +class InvalidAuthorizationHeader(CSRFTokenError): + pass + + +class InvalidAuthorizationPrefix(CSRFTokenError): + pass + + +class InvalidAuthorizationToken(CSRFTokenError): + pass + + +class InvalidDatabaseFile(ValidationError): + pass + + +class ExecutableNotFound(FileNotFoundError): + pass + + +class LinkExpiredError(ValidationError): + pass + + +class InvalidKey(ValidationError): + pass diff --git a/frappe/model/document.py b/frappe/model/document.py index cdc8be82995e..41d63452297f 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -1281,6 +1281,30 @@ def get_signature(self): """Returns signature (hash) for private URL.""" return hashlib.sha224(get_datetime_str(self.creation).encode()).hexdigest() + def get_document_share_key(self, expires_on=None, no_expiry=False): + if no_expiry: + expires_on = None + + existing_key = frappe.db.exists( + "Document Share Key", + { + "reference_doctype": self.doctype, + "reference_docname": self.name, + "expires_on": expires_on, + }, + ) + if existing_key: + doc = frappe.get_doc("Document Share Key", existing_key) + else: + doc = frappe.new_doc("Document Share Key") + doc.reference_doctype = self.doctype + doc.reference_docname = self.name + doc.expires_on = expires_on + doc.flags.no_expiry = no_expiry + doc.insert(ignore_permissions=True) + + return doc.key + def get_liked_by(self): liked_by = getattr(self, "_liked_by", None) if liked_by: diff --git a/frappe/public/scss/print.scss b/frappe/public/scss/print.scss index a610299159ab..7b8fcb32eb7f 100644 --- a/frappe/public/scss/print.scss +++ b/frappe/public/scss/print.scss @@ -2,12 +2,35 @@ @import './common/quill'; @import "./desk/css_variables"; +@import "./desk/variables"; +@import "~bootstrap/scss/utilities/spacing"; -// .print-format { -// .ql-snow .ql-editor { -// height: auto; -// min-height: 0; -// // max-height: 0; -// } -// } +// !! PDF Barcode hack !! +// Workaround for rendering barcodes prior to https://github.com/frappe/frappe/pull/15307 +@media print { + svg[data-barcode-value] > rect { + fill: white !important; + } + svg[data-barcode-value] > g { + fill: black !important; + } + .print-hide { + display: none !important; + } +} +.action-banner { + display: flex; + justify-content: flex-end; + padding-right: 20px; + font-size: var(--text-md); +} + +.invalid-state { + display: grid; + place-content: center; + height: 100vh; + img { + margin: auto; + } +} diff --git a/frappe/templates/print_formats/print_key_expired.html b/frappe/templates/print_formats/print_key_expired.html new file mode 100644 index 000000000000..a5841cdeee60 --- /dev/null +++ b/frappe/templates/print_formats/print_key_expired.html @@ -0,0 +1,11 @@ +
+ + + + + + +
+ {{ _("Key Expired") }} +
+
diff --git a/frappe/templates/print_formats/print_key_invalid.html b/frappe/templates/print_formats/print_key_invalid.html new file mode 100644 index 000000000000..4adb16563408 --- /dev/null +++ b/frappe/templates/print_formats/print_key_invalid.html @@ -0,0 +1,14 @@ +
+ + + + + + + + + +
+ {{ _("Key is Invalid")}} +
+
\ No newline at end of file diff --git a/frappe/utils/print_format.py b/frappe/utils/print_format.py index ae919ce8ddd9..398eaf16830e 100644 --- a/frappe/utils/print_format.py +++ b/frappe/utils/print_format.py @@ -5,7 +5,8 @@ from frappe.utils.pdf import get_pdf,cleanup from frappe.core.doctype.access_log.access_log import make_access_log -from PyPDF2 import PdfFileWriter +from frappe.utils.pdf import cleanup, get_pdf +from frappe.www.printview import validate_print_permission no_cache = 1 @@ -88,8 +89,22 @@ def read_multi_pdf(output): return filedata -@frappe.whitelist() + +@frappe.whitelist(allow_guest=True) def download_pdf(doctype, name, format=None, doc=None, no_letterhead=0): + doc = frappe.get_doc(doctype, name) + doc.doctype = doctype + try: + validate_print_permission(doc) + except frappe.exceptions.LinkExpiredError: + frappe.local.response.http_status_code = 410 + frappe.local.response.message = _("Link Expired") + return + except frappe.exceptions.InvalidKey: + frappe.local.response.http_status_code = 401 + frappe.local.response.message = _("Invalid Key") + return + html = frappe.get_print(doctype, name, format, doc=doc, no_letterhead=no_letterhead) frappe.local.response.filename = "{name}.pdf".format(name=name.replace(" ", "-").replace("/", "-")) frappe.local.response.filecontent = get_pdf(html) diff --git a/frappe/www/printview.html b/frappe/www/printview.html index 40fa55342a03..7d349542b1a2 100644 --- a/frappe/www/printview.html +++ b/frappe/www/printview.html @@ -6,15 +6,29 @@ {{ title }} {{ include_style("printview.css") }} - + {% if print_style %} + + {% endif %} + {% if is_invalid_print %} + {{ body }} + {% else %} +
+ + {{ _("Print") }} + + + {{ _('Get PDF') }} + +
-
- {{ body }} -
+
+ {{ body }} +
+ {% endif %} {%- if comment -%} {%- endif -%} - + \ No newline at end of file diff --git a/frappe/www/printview.py b/frappe/www/printview.py index 783ef2944e4c..90007ff8cbe6 100644 --- a/frappe/www/printview.py +++ b/frappe/www/printview.py @@ -41,16 +41,39 @@ def get_context(context): make_access_log(doctype=frappe.form_dict.doctype, document=frappe.form_dict.name, file_type='PDF', method='Print') + is_invalid_print = False + print_style = None + try: + body = get_rendered_template( + doc, + print_format=print_format, + meta=meta, + trigger_print=frappe.form_dict.trigger_print, + no_letterhead=frappe.form_dict.no_letterhead, + letterhead=letterhead, + settings=settings, + ) + print_style = get_print_style(frappe.form_dict.style, print_format) + except frappe.exceptions.LinkExpiredError: + body = frappe.get_template("templates/print_formats/print_key_expired.html").render({}) + context.http_status_code = 410 + is_invalid_print = True + except frappe.exceptions.InvalidKey: + body = frappe.get_template("templates/print_formats/print_key_invalid.html").render({}) + context.http_status_code = 401 + is_invalid_print = True + return { - "body": get_rendered_template(doc, print_format = print_format, - meta=meta, trigger_print = frappe.form_dict.trigger_print, - no_letterhead=frappe.form_dict.no_letterhead, letterhead=letterhead, - settings=settings), - "css": get_print_style(frappe.form_dict.style, print_format), + "body": body, + "print_style": print_style, "comment": frappe.session.user, - "title": doc.get(meta.title_field) if meta.title_field else doc.name, + "title": frappe.utils.strip_html(doc.get_title()), "lang": frappe.local.lang, - "layout_direction": "rtl" if is_rtl() else "ltr" + "layout_direction": "rtl" if is_rtl() else "ltr", + "is_invalid_print": is_invalid_print, + "doctype": frappe.form_dict.doctype, + "name": frappe.form_dict.name, + "key": frappe.form_dict.get("key"), } def get_print_format_doc(print_format_name, meta): @@ -230,14 +253,33 @@ def get_rendered_raw_commands(doc, name=None, print_format=None, meta=None, lang } def validate_print_permission(doc): - if frappe.form_dict.get("key"): - if frappe.form_dict.key == doc.get_signature(): + for ptype in ("read", "print"): + if frappe.has_permission(doc.doctype, ptype, doc) or frappe.has_website_permission(doc): return - for ptype in ("read", "print"): - if (not frappe.has_permission(doc.doctype, ptype, doc) - and not frappe.has_website_permission(doc)): - raise frappe.PermissionError(_("No {0} permission").format(ptype)) + key = frappe.form_dict.get("key") + if key: + validate_key(key, doc) + else: + raise frappe.PermissionError(_("You do not have permission to view this document")) + + +def validate_key(key, doc): + document_share_key = frappe.db.exists( + "Document Share Key", + {"reference_doctype": doc.doctype, "reference_docname": doc.name, "key": key}, + cache=True, + ) + if document_share_key: + if frappe.get_cached_doc("Document Share Key", document_share_key).is_expired(): + raise frappe.exceptions.LinkExpiredError + else: + return + + # TODO: Deprecate this! kept it for backward compatibility + if frappe.get_system_settings("allow_older_web_view_links") and key == doc.get_signature(): + return + raise frappe.exceptions.InvalidKey def get_letter_head(doc, no_letterhead, letterhead=None): if no_letterhead: From 84eec13338f6f293573a628dcd54eeb93a6549fb Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 29 Jun 2022 11:52:49 +0530 Subject: [PATCH 18/20] fix: remove bare exceptions --- frappe/build.py | 2 +- frappe/commands/scheduler.py | 6 ++-- frappe/core/doctype/communication/email.py | 2 +- .../doctype/data_import_legacy/importer.py | 2 +- frappe/core/doctype/doctype/test_doctype.py | 11 +++++-- frappe/core/doctype/user/user.py | 2 +- .../data_migration_connector.py | 4 +-- frappe/database/database.py | 2 +- .../doctype/system_console/system_console.py | 4 +-- frappe/email/__init__.py | 32 ++++++++----------- .../doctype/notification/notification.py | 4 +-- frappe/email/receive.py | 10 +++--- frappe/installer.py | 6 ++-- .../razorpay_settings/razorpay_settings.py | 10 +++--- frappe/model/mapper.py | 2 +- .../generate_theme_files_in_public_folder.py | 4 +-- frappe/twofactor.py | 2 +- frappe/utils/background_jobs.py | 2 +- frappe/utils/data.py | 12 ++++--- frappe/utils/pdf.py | 2 +- frappe/utils/reset_doc.py | 4 +-- frappe/utils/scheduler.py | 2 +- frappe/website/context.py | 2 +- 23 files changed, 68 insertions(+), 61 deletions(-) diff --git a/frappe/build.py b/frappe/build.py index 911a52675fe9..ef4a05d9772d 100644 --- a/frappe/build.py +++ b/frappe/build.py @@ -202,7 +202,7 @@ def symlink(target, link_name, overwrite=False): os.replace(temp_link_name, link_name) except AttributeError: os.renames(temp_link_name, link_name) - except: + except Exception: if os.path.islink(temp_link_name): os.remove(temp_link_name) raise diff --git a/frappe/commands/scheduler.py b/frappe/commands/scheduler.py index e9638800cdda..5fa5634f044a 100755 --- a/frappe/commands/scheduler.py +++ b/frappe/commands/scheduler.py @@ -10,8 +10,10 @@ def _is_scheduler_enabled(): enable_scheduler = False try: frappe.connect() - enable_scheduler = cint(frappe.db.get_single_value("System Settings", "enable_scheduler")) and True or False - except: + enable_scheduler = ( + cint(frappe.db.get_single_value("System Settings", "enable_scheduler")) and True or False + ) + except Exception: pass finally: frappe.db.close() diff --git a/frappe/core/doctype/communication/email.py b/frappe/core/doctype/communication/email.py index d3743e83cc6a..6459f0fb1088 100755 --- a/frappe/core/doctype/communication/email.py +++ b/frappe/core/doctype/communication/email.py @@ -534,7 +534,7 @@ def sendmail(communication_name, print_html=None, print_format=None, attachments else: break - except: + except Exception: traceback = frappe.log_error("frappe.core.doctype.communication.email.sendmail") raise diff --git a/frappe/core/doctype/data_import_legacy/importer.py b/frappe/core/doctype/data_import_legacy/importer.py index 35569c71860a..b3b7b53aa875 100644 --- a/frappe/core/doctype/data_import_legacy/importer.py +++ b/frappe/core/doctype/data_import_legacy/importer.py @@ -301,7 +301,7 @@ def attach_file_to_doc(doctype, docname, file_url): try: doctype = get_header_row(get_data_keys_definition().main_table)[1] columns = filter_empty_columns(get_header_row(get_data_keys_definition().columns)[1:]) - except: + except Exception: frappe.throw(_("Cannot change header content")) doctypes = [] column_idx_to_fieldname = {} diff --git a/frappe/core/doctype/doctype/test_doctype.py b/frappe/core/doctype/doctype/test_doctype.py index 3e7da6739e50..f1237c2356d6 100644 --- a/frappe/core/doctype/doctype/test_doctype.py +++ b/frappe/core/doctype/doctype/test_doctype.py @@ -239,9 +239,14 @@ def test_sync_field_order(self): test_doctype.save() test_doctype_json = frappe.get_file_json(path) - self.assertListEqual([f['fieldname'] for f in test_doctype_json['fields']], ['field_1', 'field_2', 'field_4', 'field_5']) - self.assertListEqual(test_doctype_json['field_order'], ['field_4', 'field_5', 'field_1', 'field_2']) - except: + self.assertListEqual( + [f["fieldname"] for f in test_doctype_json["fields"]], + ["field_1", "field_2", "field_4", "field_5"], + ) + self.assertListEqual( + test_doctype_json["field_order"], ["field_4", "field_5", "field_1", "field_2"] + ) + except Exception: raise finally: frappe.flags.allow_doctype_export = 0 diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index 7232f9610099..025b120e3308 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -550,7 +550,7 @@ def get_social_login_userid(self, provider): for p in self.social_logins: if p.provider == provider: return p.userid - except: + except Exception: return None def set_social_login_userid(self, provider, userid, username=None): diff --git a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.py b/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.py index 793dfe669465..bf50e95822fe 100644 --- a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.py +++ b/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.py @@ -18,8 +18,8 @@ def validate(self): if self.python_module: try: get_connection_class(self.python_module) - except: - frappe.throw(frappe._('Invalid module path')) + except Exception: + frappe.throw(frappe._("Invalid module path")) def get_connection(self): if self.python_module: diff --git a/frappe/database/database.py b/frappe/database/database.py index 8e68d0edabb8..bb96f3bd4960 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -222,7 +222,7 @@ def mogrify(self, query, values): else: try: return self._cursor.mogrify(query, values) - except: # noqa: E722 + except Exception: return (query, values) def explain_query(self, query, values=None): diff --git a/frappe/desk/doctype/system_console/system_console.py b/frappe/desk/doctype/system_console/system_console.py index 6c87ca8c36be..9abcc8d99b24 100644 --- a/frappe/desk/doctype/system_console/system_console.py +++ b/frappe/desk/doctype/system_console/system_console.py @@ -16,8 +16,8 @@ def run(self): try: frappe.debug_log = [] safe_exec(self.console) - self.output = '\n'.join(frappe.debug_log) - except: # noqa: E722 + self.output = "\n".join(frappe.debug_log) + except Exception: self.output = frappe.get_traceback() if self.commit: diff --git a/frappe/email/__init__.py b/frappe/email/__init__.py index b05aef7639ea..718ab01f00bf 100644 --- a/frappe/email/__init__.py +++ b/frappe/email/__init__.py @@ -16,24 +16,20 @@ def get_contact_list(txt, page_length=20): if cached_contacts: return cached_contacts[:page_length] - try: - match_conditions = build_match_conditions('Contact') - match_conditions = "and {0}".format(match_conditions) if match_conditions else "" - - out = frappe.db.sql("""select email_id as value, - concat(first_name, ifnull(concat(' ',last_name), '' )) as description - from tabContact - where name like %(txt)s or email_id like %(txt)s - %(condition)s - limit %(page_length)s""", { - 'txt': '%' + txt + '%', - 'condition': match_conditions, - 'page_length': page_length - }, as_dict=True) - out = filter(None, out) - - except: - raise + match_conditions = build_match_conditions("Contact") + match_conditions = "and {0}".format(match_conditions) if match_conditions else "" + + out = frappe.db.sql( + """select email_id as value, + concat(first_name, ifnull(concat(' ',last_name), '' )) as description + from tabContact + where name like %(txt)s or email_id like %(txt)s + %(condition)s + limit %(page_length)s""", + {"txt": "%" + txt + "%", "condition": match_conditions, "page_length": page_length}, + as_dict=True, + ) + out = filter(None, out) update_contact_cache(out) diff --git a/frappe/email/doctype/notification/notification.py b/frappe/email/doctype/notification/notification.py index ed7b28f53e21..48ce8d0643a7 100644 --- a/frappe/email/doctype/notification/notification.py +++ b/frappe/email/doctype/notification/notification.py @@ -136,8 +136,8 @@ def send(self, doc): if self.channel == 'System Notification' or self.send_system_notification: self.create_system_notification(doc, context) - except: - frappe.log_error(title='Failed to send notification', message=frappe.get_traceback()) + except Exception: + frappe.log_error(title="Failed to send notification", message=frappe.get_traceback()) if self.set_property_after_alert: allow_update = True diff --git a/frappe/email/receive.py b/frappe/email/receive.py index ea0c6673055d..4d3acc67e40a 100644 --- a/frappe/email/receive.py +++ b/frappe/email/receive.py @@ -359,8 +359,8 @@ def make_error_msg(self, msg_num, incoming_mail): if not incoming_mail: try: # retrieve headers - incoming_mail = Email(b'\n'.join(self.pop.top(msg_num, 5)[1])) - except: + incoming_mail = Email(b"\n".join(self.pop.top(msg_num, 5)[1])) + except Exception: pass if incoming_mail: @@ -418,8 +418,8 @@ def __init__(self, content): try: utc = email.utils.mktime_tz(email.utils.parsedate_tz(self.mail["Date"])) utc_dt = datetime.datetime.utcfromtimestamp(utc) - self.date = convert_utc_to_user_timezone(utc_dt).strftime('%Y-%m-%d %H:%M:%S') - except: + self.date = convert_utc_to_user_timezone(utc_dt).strftime("%Y-%m-%d %H:%M:%S") + except Exception: self.date = now() else: self.date = now() @@ -543,7 +543,7 @@ def get_attachment(self, part): try: fname = fname.replace('\n', ' ').replace('\r', '') fname = cstr(decode_header(fname)[0][0]) - except: + except Exception: fname = get_random_filename(content_type=content_type) else: fname = get_random_filename(content_type=content_type) diff --git a/frappe/installer.py b/frappe/installer.py index 6b1390fccbf6..46378956ed9e 100755 --- a/frappe/installer.py +++ b/frappe/installer.py @@ -501,7 +501,7 @@ def extract_sql_gzip(sql_gz_path): decompressed_file = original_file.rstrip(".gz") cmd = 'gzip -dvf < {0} > {1}'.format(original_file, decompressed_file) subprocess.check_call(cmd, shell=True) - except: + except Exception: raise return decompressed_file @@ -529,8 +529,8 @@ def extract_files(site_name, file_path): if file_path.endswith(".tar"): subprocess.check_output(['tar', 'xvf', tar_path, '--strip', '2'], cwd=abs_site_path) elif file_path.endswith(".tgz"): - subprocess.check_output(['tar', 'zxvf', tar_path, '--strip', '2'], cwd=abs_site_path) - except: + subprocess.check_output(["tar", "zxvf", tar_path, "--strip", "2"], cwd=abs_site_path) + except Exception: raise finally: frappe.destroy() diff --git a/frappe/integrations/doctype/razorpay_settings/razorpay_settings.py b/frappe/integrations/doctype/razorpay_settings/razorpay_settings.py index af7686c9b005..b9dc5846924f 100644 --- a/frappe/integrations/doctype/razorpay_settings/razorpay_settings.py +++ b/frappe/integrations/doctype/razorpay_settings/razorpay_settings.py @@ -127,9 +127,9 @@ def setup_addon(self, settings, **kwargs): "content-type": "application/json" } ) - if not resp.get('id'): - frappe.log_error(str(resp), 'Razorpay Failed while creating subscription') - except: + if not resp.get("id"): + frappe.log_error(str(resp), "Razorpay Failed while creating subscription") + except Exception: frappe.log_error(frappe.get_traceback()) # failed pass @@ -170,7 +170,7 @@ def setup_subscription(self, settings, **kwargs): else: frappe.log_error(str(resp), 'Razorpay Failed while creating subscription') - except: + except Exception: frappe.log_error(frappe.get_traceback()) # failed pass @@ -264,7 +264,7 @@ def authorize_payment(self): else: frappe.log_error(str(resp), 'Razorpay Payment not authorized') - except: + except Exception: frappe.log_error(frappe.get_traceback()) # failed pass diff --git a/frappe/model/mapper.py b/frappe/model/mapper.py index d3014435e03d..0f6f024ffba9 100644 --- a/frappe/model/mapper.py +++ b/frappe/model/mapper.py @@ -208,7 +208,7 @@ def map_fetch_fields(target_doc, df, no_copy_fields): if not linked_doc: try: linked_doc = frappe.get_doc(df.options, target_doc.get(df.fieldname)) - except: + except Exception: return val = linked_doc.get(source_fieldname) diff --git a/frappe/patches/v13_0/generate_theme_files_in_public_folder.py b/frappe/patches/v13_0/generate_theme_files_in_public_folder.py index bcb47bec24e0..599e667de70f 100644 --- a/frappe/patches/v13_0/generate_theme_files_in_public_folder.py +++ b/frappe/patches/v13_0/generate_theme_files_in_public_folder.py @@ -15,6 +15,6 @@ def execute(): try: doc.generate_bootstrap_theme() doc.save() - except: # noqa: E722 - print('Ignoring....') + except Exception: + print("Ignoring....") print(frappe.get_traceback()) diff --git a/frappe/twofactor.py b/frappe/twofactor.py index 465574791af4..cc857aae6ca6 100644 --- a/frappe/twofactor.py +++ b/frappe/twofactor.py @@ -267,7 +267,7 @@ def send_token_via_sms(otpsecret, token=None, phone_no=None): '''Send token as sms to user.''' try: from frappe.core.doctype.sms_settings.sms_settings import send_request - except: + except Exception: return False if not phone_no: diff --git a/frappe/utils/background_jobs.py b/frappe/utils/background_jobs.py index f66081e4b47d..a3c75ce7df28 100755 --- a/frappe/utils/background_jobs.py +++ b/frappe/utils/background_jobs.py @@ -129,7 +129,7 @@ def execute_job(site, method, event, job_name, kwargs, user=None, is_async=True, frappe.log_error(title=method_name) raise - except: + except Exception: frappe.db.rollback() frappe.log_error(title=method_name) frappe.db.commit() diff --git a/frappe/utils/data.py b/frappe/utils/data.py index d8bc91824b41..99a707275838 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -670,8 +670,10 @@ def floor(s): number representing the largest integer less than or equal to the specified number """ - try: num = cint(math.floor(flt(s))) - except: num = 0 + try: + num = cint(math.floor(flt(s))) + except Exception: + num = 0 return num def ceil(s): @@ -689,8 +691,10 @@ def ceil(s): smallest integer greater than or equal to the given number """ - try: num = cint(math.ceil(flt(s))) - except: num = 0 + try: + num = cint(math.ceil(flt(s))) + except Exception: + num = 0 return num def cstr(s, encoding='utf-8'): diff --git a/frappe/utils/pdf.py b/frappe/utils/pdf.py index f3044167376d..cb2b222c4570 100644 --- a/frappe/utils/pdf.py +++ b/frappe/utils/pdf.py @@ -165,7 +165,7 @@ def read_options_from_html(html): match = pattern.findall(html) if match: options[attr] = str(match[-1][3]).strip() - except: + except Exception: pass return str(soup), options diff --git a/frappe/utils/reset_doc.py b/frappe/utils/reset_doc.py index 2119df589799..503318cd8a10 100755 --- a/frappe/utils/reset_doc.py +++ b/frappe/utils/reset_doc.py @@ -30,8 +30,8 @@ def reset_doc(doctype): git_link = '/'.join(['https://raw.githubusercontent.com/frappe',\ app, branch, doc_path.split('apps/'+app)[1]]) original_file = urlopen(git_link).read() - except: - print('Did not find {0} in {1}'.format(doctype, app)) + except Exception: + print("Did not find {0} in {1}".format(doctype, app)) return # load local and original json objects diff --git a/frappe/utils/scheduler.py b/frappe/utils/scheduler.py index c439582f0a82..4ada1b74d6ca 100755 --- a/frappe/utils/scheduler.py +++ b/frappe/utils/scheduler.py @@ -72,7 +72,7 @@ def log_and_raise(): frappe.logger("scheduler").debug('Access denied for site {0}'.format(site)) else: log_and_raise() - except: + except Exception: log_and_raise() finally: diff --git a/frappe/website/context.py b/frappe/website/context.py index c898d39869e2..a2bf111c7b95 100644 --- a/frappe/website/context.py +++ b/frappe/website/context.py @@ -60,7 +60,7 @@ def update_controller_context(context, controller): context.update(ret) except (frappe.PermissionError, frappe.PageDoesNotExistError, frappe.Redirect): raise - except: + except Exception: if not any([frappe.flags.in_migrate, frappe.flags.in_website_search_build]): frappe.errprint(frappe.utils.get_traceback()) From afe30a059544214ae666609243c73e7274bdf461 Mon Sep 17 00:00:00 2001 From: HENRY Florian Date: Thu, 30 Jun 2022 20:03:47 +0200 Subject: [PATCH 19/20] fix: Issue sending email with attachement (that are not added by sending users) (fix in version-12 but not in version-13) (#17293) fix: issue #17052 fix in version-12 but not in version-13 --- frappe/core/doctype/communication/email.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/core/doctype/communication/email.py b/frappe/core/doctype/communication/email.py index 6459f0fb1088..e07e9f875670 100755 --- a/frappe/core/doctype/communication/email.py +++ b/frappe/core/doctype/communication/email.py @@ -317,7 +317,7 @@ def prepare_to_notify(doc, print_html=None, print_format=None, attachments=None) # is it a filename? try: # check for both filename and file id - file_id = frappe.db.get_list('File', or_filters={'file_name': a, 'name': a}, limit=1) + file_id = frappe.db.get_all("File", or_filters={"file_name": a, "name": a}, limit=1) if not file_id: frappe.throw(_("Unable to find attachment {0}").format(a)) file_id = file_id[0]['name'] From bf5a4e08169e8ad9a31992588c5699578c2cb0a6 Mon Sep 17 00:00:00 2001 From: Daizy Modi Date: Tue, 6 Dec 2022 11:12:38 +0530 Subject: [PATCH 20/20] fix: type conversion for read receipt in communication email (cherry picked from commit d76b5fb7194dfee6bf695c9576f6cd0795dbad39) --- frappe/core/doctype/communication/email.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/core/doctype/communication/email.py b/frappe/core/doctype/communication/email.py index e07e9f875670..bc0b9882bff1 100755 --- a/frappe/core/doctype/communication/email.py +++ b/frappe/core/doctype/communication/email.py @@ -76,7 +76,7 @@ def make(doctype=None, name=None, content=None, subject=None, sent_or_received = send_me_a_copy=cint(send_me_a_copy), cc=cc, bcc=bcc, - read_receipt=read_receipt, + read_receipt=cint(read_receipt), print_letterhead=print_letterhead, email_template=email_template, communication_type=communication_type,