diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f9f54e5..ee90c7c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ CHANGELOG for FlatCAM beta 14.11.2020 - upgraded the Extract Tool to allow aperture selection therefore narrowing down what apertures are the source for drills and/or soldermask openings +- some work in PDF parser to make it work more reliable (not sure if I succeeded) 13.11.2020 diff --git a/appParsers/ParsePDF.py b/appParsers/ParsePDF.py index 947c78f4..66858f42 100644 --- a/appParsers/ParsePDF.py +++ b/appParsers/ParsePDF.py @@ -19,12 +19,12 @@ import logging log = logging.getLogger('base') -class PdfParser(QtCore.QObject): +class PdfParser: - def __init__(self, app): - super().__init__() - self.app = app - self.step_per_circles = self.app.defaults["gerber_circle_steps"] + def __init__(self, units, resolution, abort): + self.step_per_circles = resolution + self.units = units + self.abort_flag = abort # detect stroke color change; it means a new object to be created self.stroke_color_re = re.compile(r'^\s*(\d+\.?\d*) (\d+\.?\d*) (\d+\.?\d*)\s*RG$') @@ -93,7 +93,7 @@ class PdfParser(QtCore.QObject): def parse_pdf(self, pdf_content): # the UNITS in PDF files are points and here we set the factor to convert them to real units (either MM or INCH) - if self.app.defaults['units'].upper() == 'MM': + if self.units.upper() == 'MM': # 1 inch = 72 points => 1 point = 1 / 72 = 0.01388888888 inch = 0.01388888888 inch * 25.4 = 0.35277777778 mm self.point_to_unit_factor = 25.4 / 72 else: @@ -161,12 +161,12 @@ class PdfParser(QtCore.QObject): lines = pdf_content.splitlines() for pline in lines: - if self.app.abort_flag: + if self.abort_flag: # graceful abort requested by the user raise grace line_nr += 1 - log.debug("line %d: %s" % (line_nr, pline)) + # log.debug("line %d: %s" % (line_nr, pline)) # COLOR DETECTION / OBJECT DETECTION match = self.stroke_color_re.search(pline) @@ -260,20 +260,20 @@ class PdfParser(QtCore.QObject): scale_geo = restored_transform[1] except IndexError: # nothing to remove - log.debug("parse_pdf() --> Nothing to restore") + # log.debug("parse_pdf() --> Nothing to restore") pass try: size = self.gs['line_width'].pop(-1) except IndexError: - log.debug("parse_pdf() --> Nothing to restore") + # log.debug("parse_pdf() --> Nothing to restore") # nothing to remove pass - log.debug( - "parse_pdf() --> Restore from GS found on line: %s --> " - "restored_offset=[%f, %f] ||| restored_scale=[%f, %f]" % - (line_nr, offset_geo[0], offset_geo[1], scale_geo[0], scale_geo[1])) + # log.debug( + # "parse_pdf() --> Restore from GS found on line: %s --> " + # "restored_offset=[%f, %f] ||| restored_scale=[%f, %f]" % + # (line_nr, offset_geo[0], offset_geo[1], scale_geo[0], scale_geo[1])) # log.debug("Restored Offset= [%f, %f]" % (offset_geo[0], offset_geo[1])) # log.debug("Restored Scale= [%f, %f]" % (scale_geo[0], scale_geo[1])) @@ -516,52 +516,68 @@ class PdfParser(QtCore.QObject): pass subpath['rectangle'] = [] - # store the found geometry - found_aperture = None + # #################################################################################################### + # ############################### store the found geometry ########################################### + # #################################################################################################### if apertures_dict: + found_aperture = None for apid in apertures_dict: # if we already have an aperture with the current size (rounded to 5 decimals) if apertures_dict[apid]['size'] == round(applied_size, 5): found_aperture = apid break + try: + if found_aperture: + ap_to_use = found_aperture + else: + ap_list = [int(k) for k in apertures_dict.keys()] + # perhaps it's the only aperture? and in that case we need to start from 10 + ap_list.remove(0) + if not ap_list: + aperture = 10 + else: + aperture = max(ap_list) + 1 + + ap_to_use = str(aperture) + apertures_dict[ap_to_use] = { + 'size': round(applied_size, 5), + 'type': 'C', + 'geometry': [] + } - if found_aperture: for pdf_geo in path_geo: if isinstance(pdf_geo, MultiPolygon): - for poly in pdf_geo: + for poly in pdf_geo.geoms: new_el = {'solid': poly, 'follow': poly.exterior} - apertures_dict[copy(found_aperture)]['geometry'].append(deepcopy(new_el)) - else: + apertures_dict[ap_to_use]['geometry'].append(deepcopy(new_el)) + elif isinstance(pdf_geo, Polygon): new_el = {'solid': pdf_geo, 'follow': pdf_geo.exterior} - apertures_dict[copy(found_aperture)]['geometry'].append(deepcopy(new_el)) - else: - if str(aperture) in apertures_dict.keys(): - aperture += 1 - apertures_dict[str(aperture)] = {} - apertures_dict[str(aperture)]['size'] = round(applied_size, 5) - apertures_dict[str(aperture)]['type'] = 'C' - apertures_dict[str(aperture)]['geometry'] = [] - for pdf_geo in path_geo: - if isinstance(pdf_geo, MultiPolygon): - for poly in pdf_geo: - new_el = {'solid': poly, 'follow': poly.exterior} - apertures_dict[str(aperture)]['geometry'].append(deepcopy(new_el)) + apertures_dict[ap_to_use]['geometry'].append(deepcopy(new_el)) else: - new_el = {'solid': pdf_geo, 'follow': pdf_geo.exterior} - apertures_dict[str(aperture)]['geometry'].append(deepcopy(new_el)) + new_el = {'solid': pdf_geo, 'follow': pdf_geo} + apertures_dict[ap_to_use]['geometry'].append(deepcopy(new_el)) + except Exception as e: + log.debug( + "line %d: %s ||| PdfParser.parse_pdf() Store Stroke geo -> %s" % (line_nr, pline, str(e)) + ) else: - apertures_dict[str(aperture)] = {} - apertures_dict[str(aperture)]['size'] = round(applied_size, 5) - apertures_dict[str(aperture)]['type'] = 'C' - apertures_dict[str(aperture)]['geometry'] = [] + apertures_dict[str(aperture)] = { + 'size': round(applied_size, 5), + 'type': 'C', + 'geometry': [] + } + for pdf_geo in path_geo: if isinstance(pdf_geo, MultiPolygon): for poly in pdf_geo: new_el = {'solid': poly, 'follow': poly.exterior} apertures_dict[str(aperture)]['geometry'].append(deepcopy(new_el)) - else: + elif isinstance(pdf_geo, Polygon): new_el = {'solid': pdf_geo, 'follow': pdf_geo.exterior} apertures_dict[str(aperture)]['geometry'].append(deepcopy(new_el)) + else: + new_el = {'solid': pdf_geo, 'follow': pdf_geo} + apertures_dict[str(aperture)]['geometry'].append(deepcopy(new_el)) continue @@ -674,54 +690,43 @@ class PdfParser(QtCore.QObject): # now that we finished searching for drill holes (this is not very precise because holes in the # polygon pours may appear as drill too, but .. hey you can't have it all ...) we add # clear_geometry - try: - for pdf_geo in path_geo: - if isinstance(pdf_geo, MultiPolygon): - for poly in pdf_geo: - new_el = {'clear': poly} - apertures_dict['0']['geometry'].append(deepcopy(new_el)) - else: - new_el = {'clear': pdf_geo} - apertures_dict['0']['geometry'].append(deepcopy(new_el)) - except KeyError: + if '0' not in apertures_dict: # in case there is no stroke width yet therefore no aperture - apertures_dict['0'] = {} - apertures_dict['0']['size'] = applied_size - apertures_dict['0']['type'] = 'C' - apertures_dict['0']['geometry'] = [] - for pdf_geo in path_geo: - if isinstance(pdf_geo, MultiPolygon): - for poly in pdf_geo: - new_el = {'clear': poly} - apertures_dict['0']['geometry'].append(deepcopy(new_el)) - else: - new_el = {'clear': pdf_geo} + apertures_dict['0'] = { + 'size': applied_size, + 'type': 'C', + 'geometry': [] + } + for pdf_geo in path_geo: + if isinstance(pdf_geo, MultiPolygon): + for poly in pdf_geo: + new_el = {'clear': poly} apertures_dict['0']['geometry'].append(deepcopy(new_el)) + else: + new_el = {'clear': pdf_geo} + apertures_dict['0']['geometry'].append(deepcopy(new_el)) + continue else: - # else, add the geometry as usual - try: - for pdf_geo in path_geo: - if isinstance(pdf_geo, MultiPolygon): - for poly in pdf_geo: - new_el = {'solid': poly, 'follow': poly.exterior} - apertures_dict['0']['geometry'].append(deepcopy(new_el)) - else: - new_el = {'solid': pdf_geo, 'follow': pdf_geo.exterior} - apertures_dict['0']['geometry'].append(deepcopy(new_el)) - except KeyError: + # else, store the Geometry as usual + + # ################################################################################################# + # ############################### store the found geometry ######################################## + # ################################################################################################# + if '0' not in apertures_dict: # in case there is no stroke width yet therefore no aperture - apertures_dict['0'] = {} - apertures_dict['0']['size'] = applied_size - apertures_dict['0']['type'] = 'C' - apertures_dict['0']['geometry'] = [] - for pdf_geo in path_geo: - if isinstance(pdf_geo, MultiPolygon): - for poly in pdf_geo: - new_el = {'solid': poly, 'follow': poly.exterior} - apertures_dict['0']['geometry'].append(deepcopy(new_el)) - else: - new_el = {'solid': pdf_geo, 'follow': pdf_geo.exterior} + apertures_dict['0'] = { + 'size': applied_size, + 'type': 'C', + 'geometry': [] + } + for pdf_geo in path_geo: + if isinstance(pdf_geo, MultiPolygon): + for poly in pdf_geo: + new_el = {'solid': poly, 'follow': poly.exterior} apertures_dict['0']['geometry'].append(deepcopy(new_el)) + else: + new_el = {'solid': pdf_geo, 'follow': pdf_geo.exterior} + apertures_dict['0']['geometry'].append(deepcopy(new_el)) continue # Fill and Stroke the path @@ -853,9 +858,11 @@ class PdfParser(QtCore.QObject): # we finished painting and also closed the path if it was the case close_subpath = True - # store the found geometry for stroking the path - found_aperture = None + # #################################################################################################### + # #################### store the found geometry for stroking the path ################################ + # #################################################################################################### if apertures_dict: + found_aperture = None for apid in apertures_dict: # if we already have an aperture with the current size (rounded to 5 decimals) if apertures_dict[apid]['size'] == round(applied_size, 5): @@ -863,30 +870,31 @@ class PdfParser(QtCore.QObject): break if found_aperture: - for pdf_geo in path_geo: - if isinstance(pdf_geo, MultiPolygon): - for poly in pdf_geo: - new_el = {'solid': poly, 'follow': poly.exterior} - apertures_dict[copy(found_aperture)]['geometry'].append(deepcopy(new_el)) - else: - new_el = {'solid': pdf_geo, 'follow': pdf_geo.exterior} - apertures_dict[copy(found_aperture)]['geometry'].append(deepcopy(new_el)) + ap_to_use = found_aperture else: - if str(aperture) in apertures_dict.keys(): - aperture += 1 - apertures_dict[str(aperture)] = { + ap_list = [int(k) for k in apertures_dict.keys()] + # perhaps it's the only aperture? and in that case we need to start from 10 + ap_list.remove(0) + if not ap_list: + aperture = 10 + else: + aperture = max(ap_list) + 1 + + ap_to_use = str(aperture) + apertures_dict[ap_to_use] = { 'size': round(applied_size, 5), 'type': 'C', 'geometry': [] } - for pdf_geo in path_geo: - if isinstance(pdf_geo, MultiPolygon): - for poly in pdf_geo: - new_el = {'solid': poly, 'follow': poly.exterior} - apertures_dict[str(aperture)]['geometry'].append(deepcopy(new_el)) - else: - new_el = {'solid': pdf_geo, 'follow': pdf_geo.exterior} - apertures_dict[str(aperture)]['geometry'].append(deepcopy(new_el)) + + for pdf_geo in path_geo: + if isinstance(pdf_geo, MultiPolygon): + for poly in pdf_geo: + new_el = {'solid': poly, 'follow': poly.exterior} + apertures_dict[ap_to_use]['geometry'].append(deepcopy(new_el)) + else: + new_el = {'solid': pdf_geo, 'follow': pdf_geo.exterior} + apertures_dict[ap_to_use]['geometry'].append(deepcopy(new_el)) else: apertures_dict[str(aperture)] = { 'size': round(applied_size, 5), @@ -903,63 +911,45 @@ class PdfParser(QtCore.QObject): new_el = {'solid': pdf_geo, 'follow': pdf_geo.exterior} apertures_dict[str(aperture)]['geometry'].append(deepcopy(new_el)) - # ############################################# ## - # store the found geometry for filling the path # - # ############################################# ## + # ##################################################################################################### + # ####################### store the found geometry for filling the path ############################### + # ##################################################################################################### # in case that a color change to white (transparent) occurred if flag_clear_geo is True: - try: - for pdf_geo in path_geo: - if isinstance(pdf_geo, MultiPolygon): - for poly in fill_geo: - new_el = {'clear': poly} - apertures_dict['0']['geometry'].append(deepcopy(new_el)) - else: - new_el = {'clear': pdf_geo} - apertures_dict['0']['geometry'].append(deepcopy(new_el)) - except KeyError: + if '0' not in apertures_dict: # in case there is no stroke width yet therefore no aperture apertures_dict['0'] = { 'size': round(applied_size, 5), 'type': 'C', 'geometry': [] } - - for pdf_geo in fill_geo: - if isinstance(pdf_geo, MultiPolygon): - for poly in pdf_geo: - new_el = {'clear': poly} - apertures_dict['0']['geometry'].append(deepcopy(new_el)) - else: - new_el = {'clear': pdf_geo} + for pdf_geo in fill_geo: + if isinstance(pdf_geo, MultiPolygon): + for poly in pdf_geo: + new_el = {'clear': poly} apertures_dict['0']['geometry'].append(deepcopy(new_el)) + else: + new_el = {'clear': pdf_geo} + apertures_dict['0']['geometry'].append(deepcopy(new_el)) + else: - try: - for pdf_geo in path_geo: - if isinstance(pdf_geo, MultiPolygon): - for poly in fill_geo: - new_el = {'solid': poly, 'follow': poly.exterior} - apertures_dict['0']['geometry'].append(deepcopy(new_el)) - else: - new_el = {'solid': pdf_geo, 'follow': pdf_geo.exterior} - apertures_dict['0']['geometry'].append(deepcopy(new_el)) - except KeyError: - # in case there is no stroke width yet therefore no aperture + # in case there is no stroke width yet therefore no aperture + if '0' not in apertures_dict: apertures_dict['0'] = { 'size': round(applied_size, 5), 'type': 'C', 'geometry': [] } - for pdf_geo in fill_geo: - if isinstance(pdf_geo, MultiPolygon): - for poly in pdf_geo: - new_el = {'solid': poly, 'follow': poly.exterior} - apertures_dict['0']['geometry'].append(deepcopy(new_el)) - else: - new_el = {'solid': pdf_geo, 'follow': pdf_geo.exterior} + for pdf_geo in fill_geo: + if isinstance(pdf_geo, MultiPolygon): + for poly in pdf_geo: + new_el = {'solid': poly, 'follow': poly.exterior} apertures_dict['0']['geometry'].append(deepcopy(new_el)) + else: + new_el = {'solid': pdf_geo, 'follow': pdf_geo.exterior} + apertures_dict['0']['geometry'].append(deepcopy(new_el)) continue @@ -979,7 +969,7 @@ class PdfParser(QtCore.QObject): if x in object_dict: object_dict.pop(x) - if self.app.abort_flag: + if self.abort_flag: # graceful abort requested by the user raise grace diff --git a/appTools/ToolPDF.py b/appTools/ToolPDF.py index 6aa7cdef..b1c6e7df 100644 --- a/appTools/ToolPDF.py +++ b/appTools/ToolPDF.py @@ -4,7 +4,6 @@ # Date: 4/23/2019 # # MIT Licence # # ########################################################## - from PyQt5 import QtWidgets, QtCore from appTool import AppTool @@ -14,6 +13,7 @@ from shapely.geometry import Point, MultiPolygon from shapely.ops import unary_union from copy import deepcopy +from io import BytesIO import zlib import re @@ -61,7 +61,9 @@ class ToolPDF(AppTool): # when empty we start the layer rendering self.parsing_promises = [] - self.parser = PdfParser(app=self.app) + self.parser = PdfParser(units=self.app.defaults['units'] , + resolution=self.app.defaults["gerber_circle_steps"], + abort=self.app.abort_flag) def run(self, toggle=True): self.app.defaults.report_usage("ToolPDF()") @@ -103,8 +105,7 @@ class ToolPDF(AppTool): for filename in filenames: if filename != '': - self.app.worker_task.emit({'fcn': self.open_pdf, - 'params': [filename]}) + self.app.worker_task.emit({'fcn': self.open_pdf, 'params': [filename]}) def open_pdf(self, filename): if not os.path.exists(filename): @@ -138,12 +139,62 @@ class ToolPDF(AppTool): stream_nr += 1 log.debug("PDF STREAM: %d\n" % stream_nr) s = s.strip(b'\r\n') + + # https://stackoverflow.com/questions/1089662/python-inflate-and-deflate-implementations + # def decompress(data): + # decompressed = zlib.decompressobj( + # -zlib.MAX_WBITS # see above + # ) + # inflated = decompressed.decompress(data) + # inflated += decompressed.flush() + # return inflated + + # Convert 2 Bytes If Python 3 + def C2BIP3(string): + if type(string) == bytes: + return string + else: + return bytes([ord(x) for x in string]) + + def inflate(data): + try: + return zlib.decompress(C2BIP3(data)) + except Exception: + if len(data) <= 10: + raise + oDecompress = zlib.decompressobj(-zlib.MAX_WBITS) + oStringIO = BytesIO() + count = 0 + for byte in C2BIP3(data): + try: + oStringIO.write(oDecompress.decompress(byte)) + count += 1 + except Exception: + break + if len(data) - count <= 2: + return oStringIO.getvalue() + else: + raise + try: - self.pdf_decompressed[short_name] += (zlib.decompress(s).decode('UTF-8') + '\r\n') + decomp = inflate(s) except Exception as e: - self.app.inform.emit('[ERROR_NOTCL] %s: %s\n%s' % (_("Failed to open"), str(filename), str(e))) - log.debug("ToolPDF.open_pdf().obj_init() --> %s" % str(e)) - return + decomp = None + log.debug("ToolPDF.open_pdf() -> inflate (decompress) -> %s" % str(e)) + + try: + self.pdf_decompressed[short_name] += (decomp.decode('UTF-8') + '\r\n') + except Exception: + try: + self.pdf_decompressed[short_name] += (decomp.decode('latin1') + '\r\n') + except Exception as e: + log.debug("ToolPDF.open_pdf() -> decoding error -> %s" % str(e)) + + if self.pdf_decompressed[short_name] == '': + self.app.inform.emit('[ERROR_NOTCL] %s: %s' % (_("Failed to open"), str(filename))) + log.debug("ToolPDF.open_pdf().obj_init() --> Empty file or error on decompression") + self.parsing_promises.remove(short_name) + return self.pdf_parsed[short_name]['pdf'] = self.parser.parse_pdf(pdf_content=self.pdf_decompressed[short_name]) # we used it, now we delete it @@ -351,7 +402,7 @@ class ToolPDF(AppTool): raise grace ap_dict = pdf_content[k] - print(k, ap_dict) + if ap_dict: layer_nr = k if k == 0: diff --git a/app_Main.py b/app_Main.py index 45f184e4..fc1824cd 100644 --- a/app_Main.py +++ b/app_Main.py @@ -286,6 +286,9 @@ class App(QtCore.QObject): self.grb_editor = None self.geo_editor = None + # when True, the app has to return from any thread + self.abort_flag = False + # ############################################################################################################ # ################# Setup the listening thread for another instance launching with args ###################### # ############################################################################################################ @@ -1219,9 +1222,6 @@ class App(QtCore.QObject): self.width = None self.height = None - # when True, the app has to return from any thread - self.abort_flag = False - # set the value used in the Windows Title self.engine = self.ui.general_defaults_form.general_app_group.ge_radio.get_value()