push code for 2.11 (#229)

* make usernames case sensitive in import
* Set user agent (#219)
* add logging to check signin failure
* add flows and --detail option to listing output
* encoding filter values to handle spaces and special chars in filters, added new --filter option to pass in un-encoded value for simpler input
* add --url, --include-all/--embedded-datasources for create/refresh/delete extract commands
* replace polling code with library call

---------

Co-authored-by: Bhuvnesh Singh <[email protected]>

Author: jacalata

WorldIndicators.tdsx

@@ -44,15 +44,14 @@ dependencies = [
     "appdirs",
     "doit",
     "ftfy",
-    "polling2",
     "pyinstaller_versionfile",
-    "requests>=2.11,<3.0",
+    "requests>=2.25,<3.0",
     "setuptools_scm",
     "types-appdirs",
     "types-mock",
     "types-requests",
     "types-setuptools",
-    "tableauserverclient>=0.19",
+    "tableauserverclient>=0.23",
     "urllib3>=1.24.3,<2.0",
 ]
 [project.optional-dependencies]

pyproject.toml

@@ -22,4 +22,4 @@ def run_command(args):
         logger = log(__class__.__name__, args.logging_level)
         logger.debug(_("tabcmd.launching"))
         session = Session()
-        session.create_session(args)
+        session.create_session(args, logger)

tabcmd/commands/auth/login_command.py

@@ -48,8 +48,8 @@ def __init__(self):
         self.timeout = None
 
         self.logging_level = "info"
-        self._read_from_json()
         self.logger = log(__name__, self.logging_level)  # instantiate here mostly for tests
+        self._read_from_json()
         self.tableau_server = None  # this one is an object that doesn't get persisted in the file
 
     # called before we connect to the server
@@ -206,7 +206,7 @@ def _create_new_connection(self) -> TSC.Server:
         return self.tableau_server
 
     def _read_existing_state(self):
-        if self._check_json():
+        if self._json_exists():
             self._read_from_json()
 
     def _print_server_info(self):
@@ -265,13 +265,13 @@ def _get_saved_credentials(self):
         return credentials
 
     # external entry point:
-    def create_session(self, args):
+    def create_session(self, args, logger):
         signed_in_object = None
         # pull out cached info from json, then overwrite with new args if available
         self._read_existing_state()
         self._update_session_data(args)
         self.logging_level = args.logging_level or self.logging_level
-        self.logger = self.logger or log(__class__.__name__, self.logging_level)
+        self.logger = logger or log(__class__.__name__, self.logging_level)
 
         credentials = None
         if args.password:
@@ -344,51 +344,70 @@ def _clear_data(self):
         self.timeout = None
 
     # json file functions ----------------------------------------------------
+    # These should be moved into a separate class
     def _get_file_path(self):
         home_path = os.path.expanduser("~")
         file_path = os.path.join(home_path, "tableau_auth.json")
         return file_path
 
     def _read_from_json(self):
-        if not self._check_json():
+        if not self._json_exists():
             return
         file_path = self._get_file_path()
-        data = {}
-        with open(str(file_path), "r") as file_contents:
-            data = json.load(file_contents)
+        content = None
         try:
-            for auth in data["tableau_auth"]:
-                self.auth_token = auth["auth_token"]
-                self.server_url = auth["server"]
-                self.site_name = auth["site_name"]
-                self.site_id = auth["site_id"]
-                self.username = auth["username"]
-                self.user_id = auth["user_id"]
-                self.token_name = auth["personal_access_token_name"]
-                self.token_value = auth["personal_access_token"]
-                self.last_login_using = auth["last_login_using"]
-                self.password_file = auth["password_file"]
-                self.no_prompt = auth["no_prompt"]
-                self.no_certcheck = auth["no_certcheck"]
-                self.certificate = auth["certificate"]
-                self.no_proxy = auth["no_proxy"]
-                self.proxy = auth["proxy"]
-                self.timeout = auth["timeout"]
-        except KeyError as e:
-            self.logger.debug(_("sessionoptions.errors.bad_password_file"), e)
-            self._remove_json()
-        except Exception as any_error:
-            self.logger.info(_("session.new_session"))
-            self._remove_json()
+            with open(str(file_path), "r") as file_contents:
+                data = json.load(file_contents)
+                content = data["tableau_auth"]
+        except json.JSONDecodeError as e:
+            self._wipe_bad_json(e, "Error reading data from session file")
+        except IOError as e:
+            self._wipe_bad_json(e, "Error reading session file")
+        except AttributeError as e:
+            self._wipe_bad_json(e, "Error parsing session details from file")
+        except Exception as e:
+            self._wipe_bad_json(e, "Unexpected error reading session details from file")
+
+        try:
+            auth = content[0]
+            self.auth_token = auth["auth_token"]
+            self.server_url = auth["server"]
+            self.site_name = auth["site_name"]
+            self.site_id = auth["site_id"]
+            self.username = auth["username"]
+            self.user_id = auth["user_id"]
+            self.token_name = auth["personal_access_token_name"]
+            self.token_value = auth["personal_access_token"]
+            self.last_login_using = auth["last_login_using"]
+            self.password_file = auth["password_file"]
+            self.no_prompt = auth["no_prompt"]
+            self.no_certcheck = auth["no_certcheck"]
+            self.certificate = auth["certificate"]
+            self.no_proxy = auth["no_proxy"]
+            self.proxy = auth["proxy"]
+            self.timeout = auth["timeout"]
+        except AttributeError as e:
+            self._wipe_bad_json(e, "Unrecognized attribute in session file")
+        except Exception as e:
+            self._wipe_bad_json(e, "Failed to load session file")
 
-    def _check_json(self):
+    def _wipe_bad_json(self, e, message):
+        self.logger.debug(message + ": " + e.__str__())
+        self.logger.info(_("session.new_session"))
+        self._remove_json()
+
+    def _json_exists(self):
+        # todo: make this location configurable
         home_path = os.path.expanduser("~")
         file_path = os.path.join(home_path, "tableau_auth.json")
         return os.path.exists(file_path)
 
     def _save_session_to_json(self):
-        data = self._serialize_for_save()
-        self._save_file(data)
+        try:
+            data = self._serialize_for_save()
+            self._save_file(data)
+        except Exception as e:
+            self._wipe_bad_json(e, "Failed to save session file")
 
     def _save_file(self, data):
         file_path = self._get_file_path()
@@ -420,7 +439,15 @@ def _serialize_for_save(self):
         return data
 
     def _remove_json(self):
-        file_path = self._get_file_path()
-        self._save_file({})
-        if os.path.exists(file_path):
-            os.remove(file_path)
+        file_path = ""
+        try:
+            if not self._json_exists():
+                return
+            file_path = self._get_file_path()
+            self._save_file({})
+            if os.path.exists(file_path):
+                os.remove(file_path)
+        except Exception as e:
+            message = "Error clearing session data from {}: check and remove manually".format(file_path)
+            self.logger.error(message)
+            self.logger.error(e)

tabcmd/commands/auth/session.py

@@ -1,8 +1,6 @@
 import inspect
 import sys
 
-from tableauserverclient import ServerResponseError
-
 from tabcmd.execution.localize import _
 
 
@@ -25,20 +23,19 @@ def is_expired_session(error):
     @staticmethod
     def is_resource_conflict(error):
         if hasattr(error, "code"):
-            return error.code == Constants.source_already_exists
+            return error.code.startswith(Constants.resource_conflict_general)
 
     @staticmethod
     def is_login_error(error):
         if hasattr(error, "code"):
             return error.code == Constants.login_error
 
-    @staticmethod
-    def is_server_response_error(error):
-        return isinstance(error, ServerResponseError)
-
     # https://gist.github.com/FredLoney/5454553
     @staticmethod
     def log_stack(logger):
+        if not logger:
+            print("logger not available: cannot show stack")
+            return
         try:
             """The log header message formatter."""
             HEADER_FMT = "Printing Call Stack at %s::%s"
@@ -49,11 +46,10 @@ def log_stack(logger):
             file, line, func = here[1:4]
             start = 0
             n_lines = 5
-            logger.trace(HEADER_FMT % (file, func))
-
-            for frame in stack[start + 2 : n_lines]:
+            logger.debug(HEADER_FMT % (file, func))
+            for frame in stack[start + 1 : n_lines]:
                 file, line, func = frame[1:4]
-                logger.trace(STACK_FMT % (file, line, func))
+                logger.debug(STACK_FMT % (file, line, func))
         except Exception as e:
             logger.info("Error printing stack trace:", e)
 
@@ -66,9 +62,9 @@ def exit_with_error(logger, message=None, exception=None):
             if exception:
                 if message:
                     logger.debug("Error message: " + message)
-                Errors.check_common_error_codes_and_explain(logger, exception)
+            Errors.check_common_error_codes_and_explain(logger, exception)
         except Exception as exc:
-            print(sys.stderr, "Error during log call from exception - {}".format(exc))
+            print(sys.stderr, "Error during log call from exception - {} {}".format(exc.__class__, message))
         try:
             logger.info("Exiting...")
         except Exception:
@@ -77,16 +73,15 @@ def exit_with_error(logger, message=None, exception=None):
 
     @staticmethod
     def check_common_error_codes_and_explain(logger, exception):
-        if Errors.is_server_response_error(exception):
-            logger.error(_("publish.errors.unexpected_server_response").format(exception))
-            if Errors.is_expired_session(exception):
-                logger.error(_("session.errors.session_expired"))
-                # TODO: add session as an argument to this method
-                #  and add the full command line as a field in Session?
-                # "session.session_expired_login"))
-                # session.renew_session
-                return
-            if exception.code == Constants.source_not_found:
-                logger.error(_("publish.errors.server_resource_not_found"), exception)
+        # most errors contain as much info in the message as we can get from the code
+        # identify any that we can add useful detail for and include them here
+        if Errors.is_expired_session(exception):
+            # catch this one so we can attempt to refresh the session before telling them it failed
+            logger.error(_("session.errors.session_expired"))
+            # TODO: add session as an argument to this method
+            #  and add the full command line as a field in Session?
+            # "session.session_expired_login"))
+            # session.renew_session()
+            return
         else:
             logger.error(exception)

tabcmd/commands/constants.py

@@ -1,3 +1,5 @@
+import urllib
+
 import tableauserverclient as TSC
 
 from tabcmd.commands.constants import Errors
@@ -23,7 +25,7 @@ def get_view_by_content_url(logger, server, view_content_url) -> TSC.ViewItem:
         try:
             req_option = TSC.RequestOptions()
             req_option.filter.add(TSC.Filter("contentUrl", TSC.RequestOptions.Operator.Equals, view_content_url))
-            logger.trace(req_option.get_query_params())
+            logger.debug(req_option.get_query_params())
             matching_views, paging = server.views.get(req_option)
         except Exception as e:
             Errors.exit_with_error(logger, e)
@@ -37,7 +39,7 @@ def get_wb_by_content_url(logger, server, workbook_content_url) -> TSC.WorkbookI
         try:
             req_option = TSC.RequestOptions()
             req_option.filter.add(TSC.Filter("contentUrl", TSC.RequestOptions.Operator.Equals, workbook_content_url))
-            logger.trace(req_option.get_query_params())
+            logger.debug(req_option.get_query_params())
             matching_workbooks, paging = server.workbooks.get(req_option)
         except Exception as e:
             Errors.exit_with_error(logger, e)
@@ -51,7 +53,7 @@ def get_ds_by_content_url(logger, server, datasource_content_url) -> TSC.Datasou
         try:
             req_option = TSC.RequestOptions()
             req_option.filter.add(TSC.Filter("contentUrl", TSC.RequestOptions.Operator.Equals, datasource_content_url))
-            logger.trace(req_option.get_query_params())
+            logger.debug(req_option.get_query_params())
             matching_datasources, paging = server.datasources.get(req_option)
         except Exception as e:
             Errors.exit_with_error(logger, e)
@@ -60,39 +62,52 @@ def get_ds_by_content_url(logger, server, datasource_content_url) -> TSC.Datasou
         return matching_datasources[0]
 
     @staticmethod
-    def apply_values_from_url_params(request_options: TSC.PDFRequestOptions, url, logger) -> None:
-        # should be able to replace this with request_options._append_view_filters(params)
+    def apply_values_from_url_params(logger, request_options: TSC.PDFRequestOptions, url) -> None:
         logger.debug(url)
         try:
             if "?" in url:
                 query = url.split("?")[1]
-                logger.trace("Query parameters: {}".format(query))
+                logger.debug("Query parameters: {}".format(query))
             else:
                 logger.debug("No query parameters present in url")
                 return
 
             params = query.split("&")
-            logger.trace(params)
+            logger.debug(params)
             for value in params:
                 if value.startswith(":"):
-                    DatasourcesAndWorkbooks.apply_option_value(request_options, value, logger)
+                    DatasourcesAndWorkbooks.apply_options_in_url(logger, request_options, value)
                 else:  # it must be a filter
-                    DatasourcesAndWorkbooks.apply_filter_value(request_options, value, logger)
+                    DatasourcesAndWorkbooks.apply_encoded_filter_value(logger, request_options, value)
 
         except Exception as e:
             logger.warn("Error building filter params", e)
             # ExportCommand.log_stack(logger)  # type: ignore
 
+    # this is called from within from_url_params, for each view_filter value
+    @staticmethod
+    def apply_encoded_filter_value(logger, request_options, value):
+        # the REST API doesn't appear to have the option to disambiguate with "Parameters.<fieldname>"
+        value = value.replace("Parameters.", "")
+        # the filter values received from the url are already url encoded. tsc will encode them again.
+        # so we run url.decode, which will be a no-op if they are not encoded.
+        decoded_value = urllib.parse.unquote(value)
+        logger.debug("url had `{0}`, saved as `{1}`".format(value, decoded_value))
+        DatasourcesAndWorkbooks.apply_filter_value(logger, request_options, decoded_value)
+
+    # this is called for each filter value,
+    # from apply_options, which expects an un-encoded input,
+    # or from apply_url_params via apply_encoded_filter_value which decodes the input
     @staticmethod
-    def apply_filter_value(request_options: TSC.PDFRequestOptions, value: str, logger) -> None:
-        # todo: do we need to strip Parameters.x -> x?
-        logger.trace("handling filter param {}".format(value))
+    def apply_filter_value(logger, request_options: TSC.PDFRequestOptions, value: str) -> None:
+        logger.debug("handling filter param {}".format(value))
         data_filter = value.split("=")
         request_options.vf(data_filter[0], data_filter[1])
 
+    # this is called from within from_url_params, for each param value
     @staticmethod
-    def apply_option_value(request_options: TSC.PDFRequestOptions, value: str, logger) -> None:
-        logger.trace("handling url option {}".format(value))
+    def apply_options_in_url(logger, request_options: TSC.PDFRequestOptions, value: str) -> None:
+        logger.debug("handling url option {}".format(value))
         setting = value.split("=")
         if ":iid" == setting[0]:
             logger.debug(":iid value ignored in url")
@@ -111,19 +126,18 @@ def is_truthy(value: str):
         return value.lower() in ["yes", "y", "1", "true"]
 
     @staticmethod
-    def apply_png_options(request_options: TSC.ImageRequestOptions, args, logger):
+    def apply_png_options(logger, request_options: TSC.ImageRequestOptions, args):
         if args.height or args.width:
-            # only applicable for png
             logger.warn("Height/width arguments not yet implemented in export")
         # Always request high-res images
         request_options.image_resolution = "high"
 
     @staticmethod
-    def apply_pdf_options(request_options: TSC.PDFRequestOptions, args, logger):
-        request_options.page_type = args.pagesize
+    def apply_pdf_options(logger, request_options: TSC.PDFRequestOptions, args):
         if args.pagelayout:
-            logger.debug("Setting page layout to: {}".format(args.pagelayout))
             request_options.orientation = args.pagelayout
+        if args.pagesize:
+            request_options.page_type = args.pagesize
 
     @staticmethod
     def save_to_data_file(logger, output, filename):

tabcmd/commands/datasources_and_workbooks/datasources_and_workbooks_command.py

@@ -32,7 +32,7 @@ def run_command(args):
         logger = log(__class__.__name__, args.logging_level)
         logger.debug(_("tabcmd.launching"))
         session = Session()
-        server = session.create_session(args)
+        server = session.create_session(args, logger)
         content_type: str = ""
         if args.workbook:
             content_type = "workbook"