-
Notifications
You must be signed in to change notification settings - Fork 51
Expand file tree
/
Copy pathvalidator.py
More file actions
607 lines (497 loc) · 21.8 KB
/
Copy pathvalidator.py
File metadata and controls
607 lines (497 loc) · 21.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#######################################################
# Class that validates the .mo file for correct
# html syntax and checks the consistency of
# experiment annotation in .mo with the .mos files.
#
# MWetter@lbl.gov 2013-05-31
# TSNouidui@lbl.gov 2017-04-25
#######################################################
#
# import from future to make Python2 behave like Python3
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future import standard_library
standard_library.install_aliases()
from builtins import *
from io import open
# end of from future import
import os
import re
class Validator(object):
""" Class that validates ``.mo`` files for the correct html syntax.
"""
def __init__(self):
# --------------------------
# Class variables
# self._libHome=os.path.abspath(".")
self._writeHTML = False
def validateHTMLInPackage(self, rootDir):
"""
This function recursively validates all ``.mo`` files
in a package.
If there is malformed html code in the ``info`` or the
``revision`` section,
then this function write the error message of tidy to the
standard output.
Note that the line number correspond to an intermediate format
(e.g., the output format of tidy), which may be different from
the ``.mo`` file.
:param rootDir: The root directory of the package.
:return: str[] Warning/error messages from tidylib.
Usage: Type
>>> import os
>>> import buildingspy.development.validator as v
>>> val = v.Validator()
>>> myMoLib = os.path.join(\
"buildingspy", "tests", "MyModelicaLibrary")
>>> # Get a list whose elements are the error strings
>>> errStr = val.validateHTMLInPackage(myMoLib)
"""
import os
errMsg = list()
# Make sure that the parameter rootDir points to a Modelica package.
topPackage = os.path.join(rootDir, "package.mo")
if not os.path.isfile(topPackage):
raise ValueError("Argument rootDir=%s is not a \
Modelica package. Expected file '%s'."
% (rootDir, topPackage))
for root, _, files in os.walk(rootDir):
for moFil in files:
# find the .mo file
if moFil.endswith('.mo'):
moFulNam = os.path.join(root, moFil)
err = self._validateHTML(moFulNam)[1]
if len(err) > 0:
# We found an error. Report it to the console.
# This may later be changed to use an error handler.
errMsg.append("[-- %s ]\n%s" % (moFulNam, err))
return errMsg
def _getInfoRevisionsHTML(self, moFile):
"""
This function returns a list that contain the html code of the
info and revision sections. Each element of the list
is a string.
:param moFile: The name of a Modelica source file.
:return: list The list of strings of the info and revisions
section.
"""
# Open file.
with open(moFile, mode="r", encoding="utf-8-sig") as f:
lines = f.readlines()
nLin = len(lines)
isTagClosed = True
entries = list()
for i in range(nLin):
if isTagClosed:
# search for opening tag
idxO = lines[i].find("<html>")
if idxO > -1:
# search for closing tag on same line as opening tag
idxC = lines[i].find("</html>")
if idxC > -1:
entries.append(lines[i][idxO + 6:idxC])
isTagClosed = True
else:
entries.append(lines[i][idxO + 6:])
isTagClosed = False
else:
# search for closing tag
idxC = lines[i].find("</html>")
if idxC == -1:
# closing tag not found, copy full line
entries.append(lines[i])
else:
# found closing tag, copy beginning of line only
entries.append(lines[i][0:idxC])
isTagClosed = True
entries.append("<h4>Revisions</h4>\n")
# search for opening tag on same line as closing tag
idxO = lines[i].find("<html>")
if idxO > -1:
entries.append(lines[i][idxO + 6:])
isTagClosed = False
return entries
def _validateHTML(self, moFile):
"""
This function validates the file ``moFile`` for correct html syntax.
:param moFile: The name of a Modelica source file.
:return: (str, str) The tidied markup [0] and warning/error
messages[1]. Warnings and errors are returned
just as tidylib returns them.
"""
from tidylib import tidy_document
entries = self._getInfoRevisionsHTML(moFile)
# Document header
header = "<?xml version='1.0' encoding='utf-8'?> \n \
<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\" \n \
\"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\"> \n \
<html xmlns=\"http://www.w3.org/1999/xhtml\"> \n \
<head> \n \
<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\" /> \n \
<title>xxx</title> \n \
</head> \n \
<body> \n \
<!-- +++++++++++++++++++++++++++++++++++++ -->\n"
body = ""
for line in entries:
body += line + '\n'
# Replace \" with "
body = body.replace('\\"', '"')
# Document footer
footer = "<!-- +++++++++++++++++++++++++++++++++++++ -->\n \
</body>\n \
</html>"
# Validate the string
document, errors = tidy_document(r"%s%s%s" % (header, body, footer),
options={'numeric-entities': 1,
'output-html': 1,
'alt-text': '',
'wrap': 72})
# Write html file.
if self._writeHTML:
htmlName = "%s%s" % (moFile[0:-2], "html")
with open(htmlName, mode="w", encoding="utf-8") as f:
f.write(document)
return (document, errors)
def _recursive_glob(self, rootdir='.', suffix=''):
"""
Return all files with given extension.
:param rootdir: Root directory.
:param suffix: File extension.
:return: List of files with given extension.
"""
return [os.path.join(rootdir, filename) for rootdir, dirnames,
filenames in os.walk(rootdir) for filename in filenames
if (filename.endswith(suffix) and
("ConvertBuildings_from" not in filename))]
def _check_experiment(self, name, val, value, model_path, mos_file):
"""
Check experiment annotation parameters in mo file.
:param name: Parameter name.
:param val: Value found in mo file.
:param value: Value found in mos file.
:param model_path: Path to mo file.
:param mos_file: Path to mos file.
:return: List of error messages.
"""
errMes = ""
if("*" in str(val)):
errMes += (
"Found mo file=" +
str(model_path) +
" with experiment annotation " +
self._capitalize_first(name) +
".\n" +
self._capitalize_first(name) +
" contains invalid expressions such as x * y. Only literal expressions are allowed " +
"by JModelica and OpenModelica unit tests.\n")
delta = abs(eval(val) - eval(value))
if (delta > 0):
errMes += ("Found mo file={!s} with experiment annotation {!s}.\n" +
"The value of {!s}={!s} is different from the (default) value={!s}" +
" found in the mos file={!s}.\n").format(model_path,
self._capitalize_first(name),
self._capitalize_first(name),
val, value, mos_file)
return errMes
def _missing_parameter(self, name, value, model_path, mos_file):
"""
Check missing experiment annotation parameter in mo file.
:param name: Parameter name.
:param value: Value found in mos file.
:param model_path: Path to mo file.
:param mos_file: Path to mos file.
:return: List of error messages.
"""
errMes = (
"Found mo file={!s} without parameter {!s} defined.\n" +
"The parameter name {!s} is defined in the mos file={!s}" +
" with the value {!s}. It must hence be defined in the mo file.\n").format(
model_path,
self._capitalize_first(name),
name,
mos_file,
value)
return errMes
def _capitalize_first(self, name):
"""
Capitalize the first letter of the given word.
Return a word with first letter capitalized.
:param name: Word to be capitalized.
:return: Word with first letter capitalized.
"""
lst = [word[0].upper() + word[1:] for word in name.split()]
return " ".join(lst)
def _missing_experiment_stoptime(self, mos_files):
"""
Check missing experiment and StopTime annotation in mo file.
Return number of mo files with experiment.
:param mos_files: List of mos files.
:return: List of error messages.
"""
errMes = ""
n_mo_files = 0
for mos_file in mos_files:
mos_path = os.path.join(os.sep, 'Resources', 'Scripts', 'Dymola')
model_path = mos_file.replace(mos_path, "")
model_path = model_path.replace(".mos", ".mo")
fm = open(model_path, "r", encoding="utf8")
model_content = fm.readlines()
Nlines = len(model_content)
foundExp = False
foundStop = False
for i in range(Nlines - 1, 0, -1):
line = model_content[i]
if "experiment(" in line.replace(" ", ""):
foundExp = True
n_mo_files += 1
if "StopTime=" in line.replace(" ", ""):
foundStop = True
if (not foundExp):
errMes += ("Found mo file={!s} without experiment annotation.\n").format(model_path)
if (not foundStop):
errMes += ("Found mo file={!s} without StopTime in experiment annotation.\n").format(
model_path)
# close and exit
fm.close()
return n_mo_files, errMes
def _separate_mos_files(self, mos_files):
"""
Return number of files with tolerance parameter
and two lists of mos files file, one with the simulateModel
and the other one with the translateModelFMU command.
:param mos_files: file path.
:return: List of error messages
and number of files with tolerance parameter,
and two lists of mos files file, one with the simulateModel
and the other one with the translateModelFMU command.
"""
mos_non_fmus = []
mos_fmus = []
n_tols = 0
n_fmus = 0
n_sim = 0
errMes = ""
for itr in mos_files:
found_sim = False
found_tol = False
f = open(itr, "r", encoding="utf8")
content = f.readlines()
i = 0
while i < len(content):
l = content[i]
if "tolerance=1" in (l.replace(" ", "")).lower():
found_tol = True
n_tols += 1
if "simulateModel(" in l.replace(" ", ""):
n_sim += 1
found_sim = True
mos_non_fmus.append(itr)
elif ("translateModelFMU" in l):
n_fmus += 1
mos_fmus.append(itr)
i += 1
f.close()
if (found_sim and not found_tol):
errMes += ("Found mos file={!s} without tolerance defined.\n" +
"A minimum tolerance of 1e-6 is required for JModelica.\n").format(itr)
return errMes, n_tols, mos_non_fmus, mos_fmus
def _check_tolerance(self, content, name, value, mos_file):
"""
Check value of tolerance in file.
:param content: file content.
:param name: variable name.
:param value: variable value.
:param mos_file: mos file.
:return: List of error messages.
"""
if (name + "=" == "tolerance=" and float(value) > 1e-6):
return self._wrong_parameter(mos_file, name, value)
else:
return ""
def _wrong_parameter(self, mos_file, name, value):
"""
Stop if invalid parameter is found.
:param mos_file: mos file.
:param name: parameter name.
:param value: parameter value.
:return: List of error messages.
"""
errMes = ""
if (name + "=" == "tolerance="):
if value is None:
errMes += (
"Found mos file={!s} without tolerance specified.\n" +
"A minimum tolerance of 1e-6 is required for JModelica for unit tests.\n").format(mos_file)
else:
if(float(value) > 1e-6):
errMes += ("Found mos file={!s} with tolerance={!s}.\n"
"The tolerance found is bigger than 1e-6, the maximum required by "
"JModelica for unit tests.\n").format(mos_file, value)
if (name + "=" == "stopTime="):
if value is None:
errMes += (
"Found mos file={!s} without stopTime specified.\n" +
"A non-null stopTime is required by OpenModelica for unit tests.\n").format(mos_file)
return errMes
def _getValue(self, name, line, fil_nam):
"""
Get value of input parameter.
:param name: Parameter name.
:param line: Line with parameter name.
:param fil_nam: File with parameter.
"""
# Split the name
value1 = line.split(name + "=")
# Split the value with potential character
value2 = value1[1].split(',')
# Split the value with potential character
value3 = value2[0].split(')')
try:
eval(value3[0])
except Exception as err:
err = "{!s}. Invalid literal found in file {!s}.\n".format(err, fil_nam)
raise ValueError(err)
# Return the value found
return value3[0]
def _wrong_literal(self, mos_file, name):
"""
Stop if invalid literal is found.
:param mos_file: mos file.
:param name: Parameter name.
:return: List of error messages.
"""
errMes = (
"Found mos file={!s} with invalid expression={!s}.\n" +
"This is not allowed for cross validation with JModelica.\n").format(
mos_file,
name +
'=' +
name)
return errMes
def _validate_experiment_setup(self, name, mos_files):
"""
Validate experiment setup.
:param name: Parameter name.
:param mos_files: List of mos files.
"""
N_mos_defect = 0
errMes = ""
j = 1
for mos_file in mos_files:
j += 1
f = open(mos_file, "r", encoding="utf8")
content = f.readlines()
found = False
i = 0
while not found and i < len(content):
l = content[i]
if "simulateModel(" in l.replace(" ", ""):
line = l
found = True
i += 1
try:
if name + "=" + name in line.replace(" ", ""):
value = name
errMes += self._wrong_literal(mos_file, name)
if name + "=" in line.replace(" ", ""):
value = self._getValue(name, line.replace(" ", ""), mos_file)
errMes += self._check_tolerance(content, name, value, mos_file)
else:
found = False
while not found and i < len(content):
line = content[i]
i += 1
if name + "=" in line.replace(" ", ""):
found = True
value = self._getValue(name, line.replace(" ", ""), mos_file)
errMes += self._check_tolerance(content, name, value, mos_file)
if name + "=" + name in line.replace(" ", ""):
value = name
errMes += self._wrong_literal(mos_file, name)
if not found:
if (name == "startTime"):
value = "0.0"
elif (name == "stopTime"):
value = "1.0"
elif(name == "tolerance"):
value = None
errMes += self._wrong_parameter(mos_file, name, value)
except AttributeError:
N_mos_defect += 1
if value is not None and value != name:
mos_path = os.path.join(os.sep, 'Resources', 'Scripts', 'Dymola')
model_path = mos_file.replace(mos_path, "")
model_path = model_path.replace(".mos", ".mo")
fm = open(model_path, "r", encoding="utf8")
model_content = fm.readlines()
Nlines = len(model_content)
found = False
foundStopExp_mo = False
foundStartExp_mo = False
foundToleranceExp_mo = False
for i in range(Nlines - 1, 0, -1):
line = model_content[i]
if "StopTime=" in line.replace(" ", ""):
foundStopExp_mo = True
if "StartTime=" in line.replace(" ", ""):
foundStartExp_mo = True
if "Tolerance=" in line.replace(" ", ""):
foundToleranceExp_mo = True
# Check if attributes StartTime/startTime are defined in mos and mo
if (name + "=" == "startTime=" and abs(eval(value)) > 0.0 and (not foundStartExp_mo)):
errMes += self._missing_parameter(name, value, model_path, mos_file)
# Check if attributes StopTime/stopTime are defined in mos and mo
if (name + "=" == "stopTime=" and abs(eval(value) - 1.0) >
0.0 and (not foundStopExp_mo)):
errMes += self._missing_parameter(name, value, model_path, mos_file)
# Check if attributes Tolerance/tolerance are defined in mos and mo
if (name + "=" == "tolerance=" and abs(eval(value)) >
0.0 and (not foundToleranceExp_mo)):
errMes += self._missing_parameter(name, value, model_path, mos_file)
for i in range(Nlines - 1, 0, -1):
line = model_content[i]
# if the lines contains experiment stop time, replace it
if self._capitalize_first(name) + "=" in line.replace(" ", "") and not found:
val = self._getValue(self._capitalize_first(
name), line.replace(" ", ""), model_path)
errMes += self._check_experiment(name, val, value, model_path, mos_file)
found = True
fm.close()
elif value == name:
errMes += self._wrong_literal(model_path, name)
f.close()
return errMes
def validateExperimentSetup(self, root_dir):
"""
Validate the experiment setup in ``.mo`` and ``.mos`` files.
:param root_dir: Root directory.
"""
# Make sure that the parameter root_dir points to a Modelica package.
errMes = ""
topPackage = os.path.join(root_dir, "package.mo")
if not os.path.isfile(topPackage):
errMes += ("Argument root_dir={!s} is not a Modelica package.\n" +
"Expected file={!s}.\n").format(root_dir, topPackage)
# Get the path to the mos files
rootPackage = os.path.join(root_dir, 'Resources', 'Scripts', 'Dymola')
# Get all mos files
mos_files = self._recursive_glob(rootPackage, '.mos')
# Split mos files which either contain simulateModel or translateModelFMU
errMesRes, n_tols, mos_non_fmus, _ = self._separate_mos_files(mos_files)
errMes += errMesRes
# Check if all .mo files contain experiment annotation
n_mo_files, errMesRes = self._missing_experiment_stoptime(mos_non_fmus)
errMes += errMesRes
# Validate model parameters
for i in ["stopTime", "tolerance", "startTime"]:
self._validate_experiment_setup(i, mos_non_fmus)
if(n_tols != n_mo_files):
errMes += ("The number of tolerances in the mos files={!s} does no match " +
"the number of mo files={!s}.\n").format(n_tols, n_mo_files)
if len(errMes)>0:
raise ValueError(errMes)