bartool/flatcam-wsl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

- some work in PDF parser to make it work more reliable (not sure if I succeeded)

Browse Source
This commit is contained in:
Marius Stanciu
2020-11-14 19:35:33 +02:00
committed by Marius
parent da462126ee
commit f81c6fd36a
4 changed files with 198 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@@ -10,6 +10,7 @@ CHANGELOG for FlatCAM beta
14.11.2020 14.11.2020
- upgraded the Extract Tool to allow aperture selection therefore narrowing down what apertures are the source for drills and/or soldermask openings - upgraded the Extract Tool to allow aperture selection therefore narrowing down what apertures are the source for drills and/or soldermask openings
- some work in PDF parser to make it work more reliable (not sure if I succeeded)
13.11.2020 13.11.2020

208
appParsers/ParsePDF.py
View File

@@ -19,12 +19,12 @@ import logging
log = logging.getLogger('base') log = logging.getLogger('base')
class PdfParser(QtCore.QObject): class PdfParser:
def __init__(self, app): def __init__(self, units, resolution, abort):
super().__init__() self.step_per_circles = resolution
self.app = app self.units = units
self.step_per_circles = self.app.defaults["gerber_circle_steps"] self.abort_flag = abort
# detect stroke color change; it means a new object to be created # detect stroke color change; it means a new object to be created
self.stroke_color_re = re.compile(r'^\s*(\d+\.?\d*) (\d+\.?\d*) (\d+\.?\d*)\s*RG$') self.stroke_color_re = re.compile(r'^\s*(\d+\.?\d*) (\d+\.?\d*) (\d+\.?\d*)\s*RG$')
@@ -93,7 +93,7 @@ class PdfParser(QtCore.QObject):
def parse_pdf(self, pdf_content): def parse_pdf(self, pdf_content):
# the UNITS in PDF files are points and here we set the factor to convert them to real units (either MM or INCH) # the UNITS in PDF files are points and here we set the factor to convert them to real units (either MM or INCH)
if self.app.defaults['units'].upper() == 'MM': if self.units.upper() == 'MM':
# 1 inch = 72 points => 1 point = 1 / 72 = 0.01388888888 inch = 0.01388888888 inch * 25.4 = 0.35277777778 mm # 1 inch = 72 points => 1 point = 1 / 72 = 0.01388888888 inch = 0.01388888888 inch * 25.4 = 0.35277777778 mm
self.point_to_unit_factor = 25.4 / 72 self.point_to_unit_factor = 25.4 / 72
else: else:
@@ -161,12 +161,12 @@ class PdfParser(QtCore.QObject):
lines = pdf_content.splitlines() lines = pdf_content.splitlines()
for pline in lines: for pline in lines:
if self.app.abort_flag: if self.abort_flag:
# graceful abort requested by the user # graceful abort requested by the user
raise grace raise grace
line_nr += 1 line_nr += 1
log.debug("line %d: %s" % (line_nr, pline)) # log.debug("line %d: %s" % (line_nr, pline))
# COLOR DETECTION / OBJECT DETECTION # COLOR DETECTION / OBJECT DETECTION
match = self.stroke_color_re.search(pline) match = self.stroke_color_re.search(pline)
@@ -260,20 +260,20 @@ class PdfParser(QtCore.QObject):
scale_geo = restored_transform[1] scale_geo = restored_transform[1]
except IndexError: except IndexError:
# nothing to remove # nothing to remove
log.debug("parse_pdf() --> Nothing to restore") # log.debug("parse_pdf() --> Nothing to restore")
pass pass
try: try:
size = self.gs['line_width'].pop(-1) size = self.gs['line_width'].pop(-1)
except IndexError: except IndexError:
log.debug("parse_pdf() --> Nothing to restore") # log.debug("parse_pdf() --> Nothing to restore")
# nothing to remove # nothing to remove
pass pass
log.debug( # log.debug(
"parse_pdf() --> Restore from GS found on line: %s --> " # "parse_pdf() --> Restore from GS found on line: %s --> "
"restored_offset=[%f, %f] ||| restored_scale=[%f, %f]" % # "restored_offset=[%f, %f] ||| restored_scale=[%f, %f]" %
(line_nr, offset_geo[0], offset_geo[1], scale_geo[0], scale_geo[1])) # (line_nr, offset_geo[0], offset_geo[1], scale_geo[0], scale_geo[1]))
# log.debug("Restored Offset= [%f, %f]" % (offset_geo[0], offset_geo[1])) # log.debug("Restored Offset= [%f, %f]" % (offset_geo[0], offset_geo[1]))
# log.debug("Restored Scale= [%f, %f]" % (scale_geo[0], scale_geo[1])) # log.debug("Restored Scale= [%f, %f]" % (scale_geo[0], scale_geo[1]))
@@ -516,51 +516,67 @@ class PdfParser(QtCore.QObject):
pass pass
subpath['rectangle'] = [] subpath['rectangle'] = []
# store the found geometry # ####################################################################################################
found_aperture = None # ############################### store the found geometry ###########################################
# ####################################################################################################
if apertures_dict: if apertures_dict:
found_aperture = None
for apid in apertures_dict: for apid in apertures_dict:
# if we already have an aperture with the current size (rounded to 5 decimals) # if we already have an aperture with the current size (rounded to 5 decimals)
if apertures_dict[apid]['size'] == round(applied_size, 5): if apertures_dict[apid]['size'] == round(applied_size, 5):
found_aperture = apid found_aperture = apid
break break
try:
if found_aperture: if found_aperture:
ap_to_use = found_aperture
else:
ap_list = [int(k) for k in apertures_dict.keys()]
# perhaps it's the only aperture? and in that case we need to start from 10
ap_list.remove(0)
if not ap_list:
aperture = 10
else:
aperture = max(ap_list) + 1
ap_to_use = str(aperture)
apertures_dict[ap_to_use] = {
'size': round(applied_size, 5),
'type': 'C',
'geometry': []
}
for pdf_geo in path_geo: for pdf_geo in path_geo:
if isinstance(pdf_geo, MultiPolygon): if isinstance(pdf_geo, MultiPolygon):
for poly in pdf_geo: for poly in pdf_geo.geoms:
new_el = {'solid': poly, 'follow': poly.exterior} new_el = {'solid': poly, 'follow': poly.exterior}
apertures_dict[copy(found_aperture)]['geometry'].append(deepcopy(new_el)) apertures_dict[ap_to_use]['geometry'].append(deepcopy(new_el))
else: elif isinstance(pdf_geo, Polygon):
new_el = {'solid': pdf_geo, 'follow': pdf_geo.exterior} new_el = {'solid': pdf_geo, 'follow': pdf_geo.exterior}
apertures_dict[copy(found_aperture)]['geometry'].append(deepcopy(new_el)) apertures_dict[ap_to_use]['geometry'].append(deepcopy(new_el))
else: else:
if str(aperture) in apertures_dict.keys(): new_el = {'solid': pdf_geo, 'follow': pdf_geo}
aperture += 1 apertures_dict[ap_to_use]['geometry'].append(deepcopy(new_el))
apertures_dict[str(aperture)] = {} except Exception as e:
apertures_dict[str(aperture)]['size'] = round(applied_size, 5) log.debug(
apertures_dict[str(aperture)]['type'] = 'C' "line %d: %s ||| PdfParser.parse_pdf() Store Stroke geo -> %s" % (line_nr, pline, str(e))
apertures_dict[str(aperture)]['geometry'] = [] )
else:
apertures_dict[str(aperture)] = {
'size': round(applied_size, 5),
'type': 'C',
'geometry': []
}
for pdf_geo in path_geo: for pdf_geo in path_geo:
if isinstance(pdf_geo, MultiPolygon): if isinstance(pdf_geo, MultiPolygon):
for poly in pdf_geo: for poly in pdf_geo:
new_el = {'solid': poly, 'follow': poly.exterior} new_el = {'solid': poly, 'follow': poly.exterior}
apertures_dict[str(aperture)]['geometry'].append(deepcopy(new_el)) apertures_dict[str(aperture)]['geometry'].append(deepcopy(new_el))
else: elif isinstance(pdf_geo, Polygon):
new_el = {'solid': pdf_geo, 'follow': pdf_geo.exterior} new_el = {'solid': pdf_geo, 'follow': pdf_geo.exterior}
apertures_dict[str(aperture)]['geometry'].append(deepcopy(new_el)) apertures_dict[str(aperture)]['geometry'].append(deepcopy(new_el))
else: else:
apertures_dict[str(aperture)] = {} new_el = {'solid': pdf_geo, 'follow': pdf_geo}
apertures_dict[str(aperture)]['size'] = round(applied_size, 5)
apertures_dict[str(aperture)]['type'] = 'C'
apertures_dict[str(aperture)]['geometry'] = []
for pdf_geo in path_geo:
if isinstance(pdf_geo, MultiPolygon):
for poly in pdf_geo:
new_el = {'solid': poly, 'follow': poly.exterior}
apertures_dict[str(aperture)]['geometry'].append(deepcopy(new_el))
else:
new_el = {'solid': pdf_geo, 'follow': pdf_geo.exterior}
apertures_dict[str(aperture)]['geometry'].append(deepcopy(new_el)) apertures_dict[str(aperture)]['geometry'].append(deepcopy(new_el))
continue continue
@@ -674,7 +690,13 @@ class PdfParser(QtCore.QObject):
# now that we finished searching for drill holes (this is not very precise because holes in the # now that we finished searching for drill holes (this is not very precise because holes in the
# polygon pours may appear as drill too, but .. hey you can't have it all ...) we add # polygon pours may appear as drill too, but .. hey you can't have it all ...) we add
# clear_geometry # clear_geometry
try: if '0' not in apertures_dict:
# in case there is no stroke width yet therefore no aperture
apertures_dict['0'] = {
'size': applied_size,
'type': 'C',
'geometry': []
}
for pdf_geo in path_geo: for pdf_geo in path_geo:
if isinstance(pdf_geo, MultiPolygon): if isinstance(pdf_geo, MultiPolygon):
for poly in pdf_geo: for poly in pdf_geo:
@@ -683,37 +705,20 @@ class PdfParser(QtCore.QObject):
else: else:
new_el = {'clear': pdf_geo} new_el = {'clear': pdf_geo}
apertures_dict['0']['geometry'].append(deepcopy(new_el)) apertures_dict['0']['geometry'].append(deepcopy(new_el))
except KeyError: continue
else:
# else, store the Geometry as usual
# #################################################################################################
# ############################### store the found geometry ########################################
# #################################################################################################
if '0' not in apertures_dict:
# in case there is no stroke width yet therefore no aperture # in case there is no stroke width yet therefore no aperture
apertures_dict['0'] = {} apertures_dict['0'] = {
apertures_dict['0']['size'] = applied_size 'size': applied_size,
apertures_dict['0']['type'] = 'C' 'type': 'C',
apertures_dict['0']['geometry'] = [] 'geometry': []
for pdf_geo in path_geo: }
if isinstance(pdf_geo, MultiPolygon):
for poly in pdf_geo:
new_el = {'clear': poly}
apertures_dict['0']['geometry'].append(deepcopy(new_el))
else:
new_el = {'clear': pdf_geo}
apertures_dict['0']['geometry'].append(deepcopy(new_el))
else:
# else, add the geometry as usual
try:
for pdf_geo in path_geo:
if isinstance(pdf_geo, MultiPolygon):
for poly in pdf_geo:
new_el = {'solid': poly, 'follow': poly.exterior}
apertures_dict['0']['geometry'].append(deepcopy(new_el))
else:
new_el = {'solid': pdf_geo, 'follow': pdf_geo.exterior}
apertures_dict['0']['geometry'].append(deepcopy(new_el))
except KeyError:
# in case there is no stroke width yet therefore no aperture
apertures_dict['0'] = {}
apertures_dict['0']['size'] = applied_size
apertures_dict['0']['type'] = 'C'
apertures_dict['0']['geometry'] = []
for pdf_geo in path_geo: for pdf_geo in path_geo:
if isinstance(pdf_geo, MultiPolygon): if isinstance(pdf_geo, MultiPolygon):
for poly in pdf_geo: for poly in pdf_geo:
@@ -853,9 +858,11 @@ class PdfParser(QtCore.QObject):
# we finished painting and also closed the path if it was the case # we finished painting and also closed the path if it was the case
close_subpath = True close_subpath = True
# store the found geometry for stroking the path # ####################################################################################################
found_aperture = None # #################### store the found geometry for stroking the path ################################
# ####################################################################################################
if apertures_dict: if apertures_dict:
found_aperture = None
for apid in apertures_dict: for apid in apertures_dict:
# if we already have an aperture with the current size (rounded to 5 decimals) # if we already have an aperture with the current size (rounded to 5 decimals)
if apertures_dict[apid]['size'] == round(applied_size, 5): if apertures_dict[apid]['size'] == round(applied_size, 5):
@@ -863,30 +870,31 @@ class PdfParser(QtCore.QObject):
break break
if found_aperture: if found_aperture:
for pdf_geo in path_geo: ap_to_use = found_aperture
if isinstance(pdf_geo, MultiPolygon):
for poly in pdf_geo:
new_el = {'solid': poly, 'follow': poly.exterior}
apertures_dict[copy(found_aperture)]['geometry'].append(deepcopy(new_el))
else: else:
new_el = {'solid': pdf_geo, 'follow': pdf_geo.exterior} ap_list = [int(k) for k in apertures_dict.keys()]
apertures_dict[copy(found_aperture)]['geometry'].append(deepcopy(new_el)) # perhaps it's the only aperture? and in that case we need to start from 10
ap_list.remove(0)
if not ap_list:
aperture = 10
else: else:
if str(aperture) in apertures_dict.keys(): aperture = max(ap_list) + 1
aperture += 1
apertures_dict[str(aperture)] = { ap_to_use = str(aperture)
apertures_dict[ap_to_use] = {
'size': round(applied_size, 5), 'size': round(applied_size, 5),
'type': 'C', 'type': 'C',
'geometry': [] 'geometry': []
} }
for pdf_geo in path_geo: for pdf_geo in path_geo:
if isinstance(pdf_geo, MultiPolygon): if isinstance(pdf_geo, MultiPolygon):
for poly in pdf_geo: for poly in pdf_geo:
new_el = {'solid': poly, 'follow': poly.exterior} new_el = {'solid': poly, 'follow': poly.exterior}
apertures_dict[str(aperture)]['geometry'].append(deepcopy(new_el)) apertures_dict[ap_to_use]['geometry'].append(deepcopy(new_el))
else: else:
new_el = {'solid': pdf_geo, 'follow': pdf_geo.exterior} new_el = {'solid': pdf_geo, 'follow': pdf_geo.exterior}
apertures_dict[str(aperture)]['geometry'].append(deepcopy(new_el)) apertures_dict[ap_to_use]['geometry'].append(deepcopy(new_el))
else: else:
apertures_dict[str(aperture)] = { apertures_dict[str(aperture)] = {
'size': round(applied_size, 5), 'size': round(applied_size, 5),
@@ -903,29 +911,19 @@ class PdfParser(QtCore.QObject):
new_el = {'solid': pdf_geo, 'follow': pdf_geo.exterior} new_el = {'solid': pdf_geo, 'follow': pdf_geo.exterior}
apertures_dict[str(aperture)]['geometry'].append(deepcopy(new_el)) apertures_dict[str(aperture)]['geometry'].append(deepcopy(new_el))
# ############################################# ## # #####################################################################################################
# store the found geometry for filling the path # # ####################### store the found geometry for filling the path ###############################
# ############################################# ## # #####################################################################################################
# in case that a color change to white (transparent) occurred # in case that a color change to white (transparent) occurred
if flag_clear_geo is True: if flag_clear_geo is True:
try: if '0' not in apertures_dict:
for pdf_geo in path_geo:
if isinstance(pdf_geo, MultiPolygon):
for poly in fill_geo:
new_el = {'clear': poly}
apertures_dict['0']['geometry'].append(deepcopy(new_el))
else:
new_el = {'clear': pdf_geo}
apertures_dict['0']['geometry'].append(deepcopy(new_el))
except KeyError:
# in case there is no stroke width yet therefore no aperture # in case there is no stroke width yet therefore no aperture
apertures_dict['0'] = { apertures_dict['0'] = {
'size': round(applied_size, 5), 'size': round(applied_size, 5),
'type': 'C', 'type': 'C',
'geometry': [] 'geometry': []
} }
for pdf_geo in fill_geo: for pdf_geo in fill_geo:
if isinstance(pdf_geo, MultiPolygon): if isinstance(pdf_geo, MultiPolygon):
for poly in pdf_geo: for poly in pdf_geo:
@@ -934,18 +932,10 @@ class PdfParser(QtCore.QObject):
else: else:
new_el = {'clear': pdf_geo} new_el = {'clear': pdf_geo}
apertures_dict['0']['geometry'].append(deepcopy(new_el)) apertures_dict['0']['geometry'].append(deepcopy(new_el))
else: else:
try:
for pdf_geo in path_geo:
if isinstance(pdf_geo, MultiPolygon):
for poly in fill_geo:
new_el = {'solid': poly, 'follow': poly.exterior}
apertures_dict['0']['geometry'].append(deepcopy(new_el))
else:
new_el = {'solid': pdf_geo, 'follow': pdf_geo.exterior}
apertures_dict['0']['geometry'].append(deepcopy(new_el))
except KeyError:
# in case there is no stroke width yet therefore no aperture # in case there is no stroke width yet therefore no aperture
if '0' not in apertures_dict:
apertures_dict['0'] = { apertures_dict['0'] = {
'size': round(applied_size, 5), 'size': round(applied_size, 5),
'type': 'C', 'type': 'C',
@@ -979,7 +969,7 @@ class PdfParser(QtCore.QObject):
if x in object_dict: if x in object_dict:
object_dict.pop(x) object_dict.pop(x)
if self.app.abort_flag: if self.abort_flag:
# graceful abort requested by the user # graceful abort requested by the user
raise grace raise grace

67
appTools/ToolPDF.py
View File

@@ -4,7 +4,6 @@
# Date: 4/23/2019 # # Date: 4/23/2019 #
# MIT Licence # # MIT Licence #
# ########################################################## # ##########################################################
from PyQt5 import QtWidgets, QtCore from PyQt5 import QtWidgets, QtCore
from appTool import AppTool from appTool import AppTool
@@ -14,6 +13,7 @@ from shapely.geometry import Point, MultiPolygon
from shapely.ops import unary_union from shapely.ops import unary_union
from copy import deepcopy from copy import deepcopy
from io import BytesIO
import zlib import zlib
import re import re
@@ -61,7 +61,9 @@ class ToolPDF(AppTool):
# when empty we start the layer rendering # when empty we start the layer rendering
self.parsing_promises = [] self.parsing_promises = []
self.parser = PdfParser(app=self.app) self.parser = PdfParser(units=self.app.defaults['units'] ,
resolution=self.app.defaults["gerber_circle_steps"],
abort=self.app.abort_flag)
def run(self, toggle=True): def run(self, toggle=True):
self.app.defaults.report_usage("ToolPDF()") self.app.defaults.report_usage("ToolPDF()")
@@ -103,8 +105,7 @@ class ToolPDF(AppTool):
for filename in filenames: for filename in filenames:
if filename != '': if filename != '':
self.app.worker_task.emit({'fcn': self.open_pdf, self.app.worker_task.emit({'fcn': self.open_pdf, 'params': [filename]})
'params': [filename]})
def open_pdf(self, filename): def open_pdf(self, filename):
if not os.path.exists(filename): if not os.path.exists(filename):
@@ -138,11 +139,61 @@ class ToolPDF(AppTool):
stream_nr += 1 stream_nr += 1
log.debug("PDF STREAM: %d\n" % stream_nr) log.debug("PDF STREAM: %d\n" % stream_nr)
s = s.strip(b'\r\n') s = s.strip(b'\r\n')
# https://stackoverflow.com/questions/1089662/python-inflate-and-deflate-implementations
# def decompress(data):
# decompressed = zlib.decompressobj(
# -zlib.MAX_WBITS # see above
# )
# inflated = decompressed.decompress(data)
# inflated += decompressed.flush()
# return inflated
# Convert 2 Bytes If Python 3
def C2BIP3(string):
if type(string) == bytes:
return string
else:
return bytes([ord(x) for x in string])
def inflate(data):
try: try:
self.pdf_decompressed[short_name] += (zlib.decompress(s).decode('UTF-8') + '\r\n') return zlib.decompress(C2BIP3(data))
except Exception:
if len(data) <= 10:
raise
oDecompress = zlib.decompressobj(-zlib.MAX_WBITS)
oStringIO = BytesIO()
count = 0
for byte in C2BIP3(data):
try:
oStringIO.write(oDecompress.decompress(byte))
count += 1
except Exception:
break
if len(data) - count <= 2:
return oStringIO.getvalue()
else:
raise
try:
decomp = inflate(s)
except Exception as e: except Exception as e:
self.app.inform.emit('[ERROR_NOTCL] %s: %s\n%s' % (_("Failed to open"), str(filename), str(e))) decomp = None
log.debug("ToolPDF.open_pdf().obj_init() --> %s" % str(e)) log.debug("ToolPDF.open_pdf() -> inflate (decompress) -> %s" % str(e))
try:
self.pdf_decompressed[short_name] += (decomp.decode('UTF-8') + '\r\n')
except Exception:
try:
self.pdf_decompressed[short_name] += (decomp.decode('latin1') + '\r\n')
except Exception as e:
log.debug("ToolPDF.open_pdf() -> decoding error -> %s" % str(e))
if self.pdf_decompressed[short_name] == '':
self.app.inform.emit('[ERROR_NOTCL] %s: %s' % (_("Failed to open"), str(filename)))
log.debug("ToolPDF.open_pdf().obj_init() --> Empty file or error on decompression")
self.parsing_promises.remove(short_name)
return return
self.pdf_parsed[short_name]['pdf'] = self.parser.parse_pdf(pdf_content=self.pdf_decompressed[short_name]) self.pdf_parsed[short_name]['pdf'] = self.parser.parse_pdf(pdf_content=self.pdf_decompressed[short_name])
@@ -351,7 +402,7 @@ class ToolPDF(AppTool):
raise grace raise grace
ap_dict = pdf_content[k] ap_dict = pdf_content[k]
print(k, ap_dict)
if ap_dict: if ap_dict:
layer_nr = k layer_nr = k
if k == 0: if k == 0:

6
app_Main.py
View File

@@ -286,6 +286,9 @@ class App(QtCore.QObject):
self.grb_editor = None self.grb_editor = None
self.geo_editor = None self.geo_editor = None
# when True, the app has to return from any thread
self.abort_flag = False
# ############################################################################################################ # ############################################################################################################
# ################# Setup the listening thread for another instance launching with args ###################### # ################# Setup the listening thread for another instance launching with args ######################
# ############################################################################################################ # ############################################################################################################
@@ -1219,9 +1222,6 @@ class App(QtCore.QObject):
self.width = None self.width = None
self.height = None self.height = None
# when True, the app has to return from any thread
self.abort_flag = False
# set the value used in the Windows Title # set the value used in the Windows Title
self.engine = self.ui.general_defaults_form.general_app_group.ge_radio.get_value() self.engine = self.ui.general_defaults_form.general_app_group.ge_radio.get_value()