$title 'Driver for quality tests of all sorts' (QUALITY,SEQ=2) $version 142 $eolCom // $escape & $if not set TRACE $set TRACE testlib.trc // gtrace file trace=%TRACE% $if not set TRACEREP $set TRACEREP testlib.rep // Gtrace report file $set TL 6 $if NOT %gams.tl%==0 $set TL %gams.tl% // the default TL of 6 is just for starters - just trying it on for size $set SLASH \ $if %system.filesys% == UNIX $set SLASH / $if not set PREFIX $set PREFIX %system.buildcode% $if not set FAIL $set ALL alltests.gms $if set FAIL $set ALL alltests.tmp $if not set FAIL $set FAIL failures_qa.gms $ifThen set GAZ $$ if not set DEMOSIZE $set DEMOSIZE 1 $$ if not set GLOBALSIZE $set GLOBALSIZE 1 $endIf $include testmod.inc SET mdl(m); SET ignore(m) models which are not supposed to be tested here / BADPT4 'is that test correct?' BADPT5 'is that test correct?' CTRLC 'interactive test' ASYNNTRP 'contains race conditions' INTERRUPT 'interactive/timing critical test' (fnacos,fnacosx,fnasin,fnasinx,fnatan,fnatan2,fnatan2n,fnatan2x,fncentr fncentr2*fncentr6,fncentrx,fncentry,fncos,fncosh,fnedist,fnentro,fnentro2 fnerrf,fnerrf2,fnexp,fnfact,fngamma,fngamma2,fnlog,fnlog10,fnlog2,fnncpcm fnncpcm2,fnncpf,fnncpf2,fnncpf3,fnncpvup,fnncpvupow2,fnncpvus,fnncpvusin2,fnncpvux,fnpi,fnpower,fnrpow fnrpow2,fnrpow3,fnsigmo,fnsigmo2,fnsin,fnsinh,fnsqr,fnsqrt,fntan,fntanh fnvcpow,fnvcpow2,fnspowx,fnspown,fnsincosintrv fnsllog10,fnsllog102,fnslexp,fnslexp2,fnslrec,fnslrec2 fnsqlog10,fnsqlog102,fnsqexp,fnsqexp2,fnsqrec,fnsqrec2 fnloggamma,fnbinomial,fnbeta,fnlogbeta,fnlogit fnlse,fnlsemax,fnlsemaxsc,fnlsemin,fnlseminsc ) '(supposed to be) tested in funcs5' (COMPLINK EX1, EX1X, EX2, EX3, EX4, EX4X, EX5, EX6 ER1, ER2, ER3 EXMCP1, EXMCP2, EXMCP3, EXMCP4, EXMCP5, EXMCP6) 'used by testexeq' ENPRO56 'Used by CTRLC' FNCVPOW FNCVPOW2 FNCVPOW3 FNEDIST2 FNEDIST3 FUZZY 'Used by CTRLC' gamsbas1 'gamsbas no longer shipped' $ifThen %DEMOSIZE%==1 kestrel1*kestrel5 $endIf GFT 'Used by CTRLC' (lp05*lp07) 'depends on reported iterations which does not work reliably for a lot of solvers' lp09 'same as lp03' math1 'an example, not a test' mcp07 'an example, not a test' mcp09 'Trac #1809 - outstanding' MOD011 'Used by CTRLC' MPSWRIT1 'mpswrite is gone' (N3707, NUCLEAR49B) 'Used by CTRLC' pgams01 'part of pgams02' (POPDYNMMCP25, POPDYNMMCP250, POPDYNMMCP1000) 'Used by CTRLC' qextract1 'about performance, not correctness' quality 'obvious' (rest2,save2) 'part of restart2' testhtm 'obvious' testutil 'obvious' tramp01 'tramp is gone' mip06 'longer-running test' $if %sysenv.GHOST% == KumaNew sql2gms1 'sql2gms cannot find "provider"' syschk1 'central premise is a race condition' /; mdl(m) = not (letters(m) or ignore(m)); SET suite / baron, cmex01, cmex02, cmex03, conopt, convert, copt, cplex, emp, examiner, gamschk, gurobi, guss, highs, kestrel, knitro, lazy, lindoglobal, lp, mip, minlp, miqcp, nlp, miles, minos, mosek, mpsge, cns, mpec, qcp, mcp, nlpec, path, pathnlp, refact, reshop, sbb, scip, sensitivity, tramp xpress /; SET type model types / CNS, DNLP, DECIS, EMP, GAMS, LP, MCP, MIP, MINLP, MIQCP MPEC, MPSGE, NLP, QCP, RMINLP, RMIP, RMIQCP /; SET solver 'master set of solvers' / system.solverNames /; SET solverPlatformMap(solver,*) / system.SolverPlatformMap /; SET noDemo(solver) 'solvers that do not work as expected in demo mode'; noDemo(solver) = [sameas(solver,'gurobi')]; noDemo(solver) = [sameas(solver,'gurobiold')]; SET avail(solver) 'Solvers available for this platform'; avail(solver) = sum(solverPlatformMap(solver,'%system.platform%'),1) and solvers(solver); $if set GAZ avail(noDemo) = no; SET filter(solver) 'solvers to filter out' / //can be overwritten by '--solver' flag, but GAMSCHK and some additional solvers are totally switched off later in the SKIP set * this first bunch includes the more obvious choices CONVERT, DE, DECISC, DECISM, EXAMINER, EXAMINER2, GAMSCHK, KESTREL GUSS, PATHVI /; set globalSolver(solver) Solvers with global demo limit / ANTIGONE, BARON, LINDOGLOBAL /; alias (solver,solvera); SET ASM(solvera,solver) 'Alias Solver Map' / CONOPT4. CONOPT GLOMIQO. ANTIGONE LOGMIP. JAMS /; SET solverAlias(solvera) 'Aliases of Solvers'; option solverAlias < ASM; $set GAMSlo %GAMS.lo% $if %GAMS.ide% == 1 $set GAMSlo 2 $set FLAGS lo=%GAMSlo% $if %DEMOSIZE% == 1 $set FLAGS --DEMOSIZE=1 %FLAGS% $if set before $set FLAGS --before=%before% %FLAGS% $if %runAsTester% == 1 $set FLAGS --noComp=1 %FLAGS% SET modelFlags(m) / assign1 'pw=79' * temporary flags below this line * CNS marginals reset with SL=4 or SL=7: gitlab issue #6463' sl4cns14 '--SKIPSL4=true --SKIPSL7=true' /; * set --QCPMCHECKS=0 if marginals are OK for QP and bad for QCP in general * set --QPMCHECKS=0 if marginals are bad for QP and bad for QCP in general SET solverFlags(solver) / alphaecp '--QPMCHECKS=0' antigone '--GLOBALSIZE=1' baron '--GLOBALSIZE=1 --TESTTOL=1e-5 --MIPMCHECKS=0 --QPMCHECKS=0 --MCHECKS=0' copt '--QCPMCHECKS=0' // no duals for quadratic equations provided by COPT so far cplex '--QCPMCHECKS=0 --TESTTOL=2e-6' // hack since it does not get marginals for QCP knitro '--TESTTOL=5e-5' * actually lindo and lindoglobal behave the same with respect to globalsize, but by setting * the flag for one but not the other we test both, the global and the local solver in demo mode lindo '--TESTTOL=1e-5' lindoglobal '--GLOBALSIZE=1 --TESTTOL=1e-5' snopt '--TESTTOL=1e-4' // marginals are not so accurate nlpec '--MCPMCHECKS=0 --NONSTRICT=0' // hack for bad marginals for MCP scip '--MIPMCHECKS=0 --QPMCHECKS=0 --MCHECKS=0 --TESTTOL=1e-3' xpress '--TESTTOL=2e-5 --QPMCHECKS=0' xpressold '--TESTTOL=2e-5 --QPMCHECKS=0' shot '--QPMCHECKS=0' highs '--MIPMCHECKS=0' /; SET modelSolverFlags(m,solver) / (lp03,sl4lp03).(conopt3,conopt,minos,mosek,quadminos,snopt) '--SKIPITER=1' lp04.knitro '--SKIPUNBND=1' lp15.(ipopt,ipopth) '--TESTTOL=1e-5' (qcp01,nlpqcp01,sl4qcp01).scip '--TESTTOL=1e-2' (qcp02,nlpqcp02,sl4qcp02).scip 'optdir=.. optfile=100' (nlpqcp01,qcp01,sl4qcp01).(xpress,xpressold) '--TESTTOL=1e-4' (cns01,cns02,sl4cns01,sl4cns02).(xpress,xpressold) '--TESTTOL=1e-3' qcp04.(xpress,xpressold) '--TESTTOL=1e-2' qcp09.(cplex,mosek) '--TESTTOL=1e-4' (qcp02,sl4qcp02).cplex '--TESTTOL=1e-2' qcp04.cplex '--TESTTOL=1e-4' (qcp02,sl4qcp02,qcp09).(gurobi,gurobiold) '--TESTTOL=1e-2 --QCPMCHECKS=0' qcp04.(gurobi,gurobiold) '--TESTTOL=1e-3 --QCPMCHECKS=0' miqcp03.(gurobi,gurobiold) '--TESTTOL=1e-3' semicon1.(gurobi,gurobiold) '--MIPMCHECKS=0' (semicon1,sl4scon1).(lindo,lindoglobal) '--MIPMCHECKS=0' (miqcp01).(lindo,lindoglobal) '--QPMCHECKS=0' miqcp03.(lindo,lindoglobal) '--QPMCHECKS=0 --TESTTOL=1e-4' // --QPMCHECKS=0 should be just temporary, see devel/products#6913 miqcp03.(mosek) '--TESTTOL=1e-4' miqcp03.shot '--TESTTOL=1e-5' qcp04.(lindo,lindoglobal) '--TESTTOL=1e-4' * test wrongly expects that solver gives same solution on different formulations, so turn off this part of the test qcp05.(lindo,lindoglobal) '--TESTTOL=100' qcp09.(lindo,lindoglobal,copt) '--TESTTOL=1e-3' nlp01.pathnlp '--TESTTOL=2e-6' nlp01.(copt,xpress,xpressold) '--MCHECKS=0' miqcp03.(xpress,xpressold) 'optdir=.. optfile=500' scensol1.snopt 'bratio=1' (miqcp01,miqcp03).knitro '--QPMCHECKS=0' (lp01,lp04,lp10,lp11,lp12,lp14,mip04,sl4lp01,sl4lp10,sl4lp11).cplex 'optdir=.. optfile=100' (lp01,lp10,lp12,lp14,mip04,sl4lp01,sl4lp10).(gurobi,gurobiold) 'optdir=.. optfile=100' * temporary flags below this line /; SET ms(m,suite) / ( baron01 * baron03 ).baron ( conopt01*conopt03,condtwin ).conopt ( convert2*convert8,convert10*convert17 ).convert (copt01*copt02).copt ( cplex01*cplex08 ).cplex ( emp01*emp35 empadj01 empbp01*empbp06 empdisj1*empdisj7 empecs01*empecs02 emplog1*emplog2 emplp01*emplp03, emplp10*emplp13 emppy1*emppy3 empsp01 empvi01*empvi05 ).emp ( examin01*examin05 ).examiner ( gamschk1*gamschk2 ).gamschk ( gurobi01,gurobi04,gurobi07 ).gurobi highs01.highs ( kestrel1*kestrel5 ).kestrel ( knitro01*knitro03 ).knitro ( lazy01 ).lazy ( lindgl01*lindgl05,lindorng,lindorg2 ).lindoglobal ( empty2, lo01, lolp,lp01*lp04,lp08,lp10*lp15,sl4lp01,sl4lp02,sl4lp03,sl4lp10,sl4lp11 ).lp ( mip01*mip06,sos1a,sos2a,semicon1,semiint1,sl4mip01,sl4sos1a,sl4scon1, indic01*indic04,dumpsol,pf4mip,sosmip01,mipstopexpr1,echoindic ).mip ( pf4minlp,sosminlp01,binary4 ).minlp ( miles01 ).miles ( minos01 ).minos ( sdp01,powercone1,powercone2,expcone1 ).mosek ( circlen,fnlseslv,lonlp,nlp01,nlplp02,nlpqcp01,nlpqcp02 ).nlp ( mpsge01*mpsge14 ).mpsge ( cns01*cns14,sl4cns01*sl4cns14 ).cns ( mpec01,mpec03,mpec04*mpec08 ).mpec ( lomcp,mcp01*mcp06,mcp08,mcp11*mcp19,sl4mcp01*sl4mcp04,sl4mcp06,scensol10 ).mcp ( miqcp01*miqcp04,sosmiqcp01 ).miqcp ( qcp01*qcp12,sl4qcp01*sl4qcp03 ).qcp ( alias01, assign1 * assign4, asynexec, attrib01*attrib02 binary1 * binary3, blank, breakcont1*breakcont2 caconcat, cacsvr, cacsvw, cadomainw, call1 * call6, caproject, cagamsrw, cagdxrw, casqlr, casqlw, cafilter, calabelm card01 * card03, carxr, capcode, carcr, caxlsr, caxlsrw choles01 * choles02 clear1, clear2, clear3, clear4, clear5 clear6, clear7 compile1 * compile8 cpplib00 * cpplib05, csvread1, csvwrite1 data1 dattim1 * dattim6 dco01 ddlist1 decis01 * decis03, decla1 * decla3, delim1 * delim5, domain01, duplic01, dump01, dumpopt1* dumpopt2, echo1, edcrypt1, eigval01, eigvec01 *eigvec02 embed1 * embed3, epstozero1, etsuf01 eval01 * eval08, execerr1 fitlib01, funcs3, funcs4, funcs5, funcback gdx1 * gdx9 gdxcomp1* gdxcomp2 gdxconv1 gdxcopy1* gdxcopy6 gdxdiff1* gdxdiff2, gdxdump1*gdxdump4, gdxlongpaths, gdxmerg1 *gdxmerg4, gdxrank1, gdxrename1, gdxrename2, gdxencoding1 gdxxrw1 * gdxxrw15 gpsollin gzip01 idir01 idxoper1, idxperm1* idxperm4 ifnot1, ifstat1 * ifstat8 ifthen1 * ifthen7 implassign01 interval invert01 *invert02 lagd1, lags1, ldidx01 * ldidx05 libdepend load1 * load18 local01 loop1, loop2, loop3 * loop10 lsalib01 macro01 * macro02, map1, match01, mdb2gms1, memmon1, moo1, mtxlib, nonneg onelab1, onelab2 onmulti1 * onmulti8 optdir1 pgams02 phantom1, phantom2, pipe1, pipe2, pipe3, pipe4, pipe5, pipe6, pipe7, pipe8 pipe9, pipe10 point1, point2 power1, power2 privat01 pwplib01 put1 * put13 restart2, sameas1, save3 scale01*scale02, scnred01 *scnred02 set1 * set11 sl601 slx01 solveopt01, sort1 sql2gms1 stolib01 single01* single05, suffix01*suffix02 symxref1, syschk2, sysenv01*sysenv02 tabsubst, tester, title1 trilib01* trilib04 trylin01 uldidx01 * uldidx03 unary1, unary2, undf1, univ1 unload1 * unload16 utils01 * utils03 warning1* warning2 zeroripple, zerotoeps1 ).cmex01 ( asyncfix01*asyncfix02, dict1, dirs01, dmpsym01 embmihws, embpy01* embpy14, embgms01*embgms06,embgms08*embgms09, empty1, encoding01, eps1, execmode01, forceerr funcs1, funcs2, gbin1, gdxsqlite1, gams2numpy01, gams2numpy02, gamsjupyter01 genfile01, goto01 handle01, handle02, mps2gms1, mps2gms2 implset1* implset2, limdom01* limdom04 nest1, nest2, nestblock01 nlcode1 * nlcode7, obfusc01, offinc01, output01*output02 perm1, platform01, prevwork1, procdir1, savep1 * savep2, show01, solver01 traceuc vardat01 *vardat05 veda1 gamsxcppmex gamsincr, gamsincr02 embmilic ).cmex02 ( badpt1, badpt2, badpt3 cerr1, cmexrc01, cmi01, idc01, ide01, dbg01, maxima, mcp10, merr1, model2tex1,mrw01,mrw02 rs01 * rs02, refact00, testexeq, shuffle1, ssuffix, studio01, xerr1, xerr2, doc01, buildwarnings, csv2gdx1*csv2gdx11 ).cmex03 (mpec02,nlpec01,nlpec02,nlpec03).nlpec ( path01 ).path ( pathnlp1, pathnlp2 ).pathnlp ( refact01*refact05 ).refact ( embrhp01*embrhp02 ).reshop ( sbb01, sbb02 ).sbb ( scip01 ).scip ( sens01 ).sensitivity ( gussskip scen01 * scen03 scenempty scensol1*scensol9 ).guss ( tramp01 ).tramp ( xpress01 * xpress08 ).xpress /; SET st(suite,type) 'default type of models unless set below' / baron.GAMS cmex01.GAMS cmex02.GAMS cmex03.GAMS conopt.NLP convert.GAMS copt.LP cplex.LP emp.EMP examiner.GAMS gamschk.GAMS gurobi.LP guss.LP highs.LP kestrel.GAMS knitro.GAMS lazy.MIP lindoglobal.GAMS lp.LP mip.MIP minlp.MINLP miles.GAMS minos.GAMS mosek.GAMS nlp.NLP mpsge.MCP cns.CNS mpec.MPEC miqcp.MIQCP qcp.QCP mcp.MCP nlpec.MPEC path.GAMS pathnlp.GAMS refact.GAMS reshop.EMP sbb.GAMS scip.MIP sensitivity.LP tramp.GAMS xpress.GAMS /; * mt can be computed based on suite for now - if we have other models that * don't have a suite membership, we can bang them in. SET mt(m,type); mt(mdl,type) = sum{ms(mdl,suite), st(suite,type)}; parameter typecount(m); typecount(mdl) = sum{mt(mdl,type),1} - 1; abort$[smax{mdl,abs(typecount(mdl))} ne 0] typecount; $onEmpty SET notQuick(m) / doc01 'linkchecker drags on and on' lp12 'lots of sleeping' nlpec01 'lots of smaller sub-jobs in this one' testexeq 'compile lots of shared libs and test with them all' /; SET notGAZ(m) / #notQuick (casqlr, casqlw) 'require credentials to access remote servers' (embpy01*embpy07) 'required libs not part of a GAZ build' embpy08 'Python API setting is required for this test' (emppy1*emppy3) 'required libs not part of a GAZ build' (gams2numpy01*gams2numpy02) 'required libs not part of a GAZ build' gamsjupyter01 'required libs not part of a GAZ build' implset1 'required libs not part of a GAZ build' libdepend 'GAZ builds need not satisfy library version requirements' lsalib01 'this test fails if you have a Lindo license' model2tex1 'required libs not part of a GAZ build' privat01 'required data is not part of a GAZ build' (rs01,rs02) 'GAZ builds need not support Gurobi remote server' scensol8 'GAZ builds need not support Gurobi remote server' gdxrename1 'not working with GAZ for Steve' gdxrename2 'not working with GAZ for Steve' (cacsvr,csv2gdx1*csv2gdx8,csvread1,csvwrite1,encoding01) 'tested functionality not part of a GAZ build' $if %system.buildcode% == WEI mdb2gms1 'test for stuff built on vs8' $if %system.buildcode% == WEI sql2gms1 'test for stuff built on vs8' $if %system.buildcode% == WEI gdxxrw1 * gdxxrw15 'test for stuff built on vs8' $if %system.buildcode% == WEI trilib02 'not working on xps15' /; SET gskip(m) 'CMEX tests we know will fail' / $ifThenI not x%sysenv.GBRANCHNAME%==xmaster $if %system.buildcode% == WEI cmexrc01 'creates a pop-up if not alpha' $endIf $if %system.buildcode% == DEG decis01 'we do not have DECIS here' $if %system.buildcode% == DEG lindgl02 'Takes long for initial solution, #6108#note_193777' minos01 'Minos cycling problem, it is just luck if this test does not fail, Trac #1700' gdxxrw14 'Jan is looking into this' $if %system.buildcode% == DAC memmon1 'memory usage stats not reasonable on DAC, #6077' /; $if set NOSKIP gskip(mdl) = no; $if set QUICK gskip(mdl)$notquick(mdl) = yes; $if set GAZ gskip(mdl)$notGAZ(mdl) = yes; SET suiteskip(solver,m) 'skips for tests in solver-specific suites' / (gurobi,gurobiold).gurobi01 'with Gurobi 5.6 the default options perform so well that we cannot get a reliable improvement with tuning for this model' /; SET skip(solver,m) 'we know these will fail' / * put permanent skips first (baron,scip).nlplp02 'does not return marginals and model is missing --*MCHECKS logic' (knitro,pathnlp).(lp03,sl4lp03) 'issue with nonbasic free vars : simplex-only test' miles.(mcp01*mcp04,sl4mcp01*sl4mcp04) 'cannot solve models with all variables fixed' (miles,nlpec).scensol10 'GUSS requires a sl=5 solver' mosek.lonlp 'Model is nonconvex' (CBC,COPT).(lp03,sl4lp03) 'Doubt if the test is correct' (Ipopt,IpoptH).(lp03,sl4lp03) 'Ipopt does not compute a basis' (gurobi,gurobiold).lp04 'Gurobi does not set inf/unb marker with (dual) infeasible models (uses Farkas certificate)' lindo.(qcp02,nlpqcp02,sl4qcp02) 'precission issue' (ipopt,ipopth).(cns12,sl4cns12) 'restoration phase fails to converge to a solution that is feasible w.r.t. tol' (xpress,xpressold).(cns12,sl4cns12) 'xpress does not converge within iteration limit' copt.(cns01,sl4cns01) 'test expects solution with INFES markers when infeasible, but COPT does not return solution if infeasible' minos.(cns01,sl4cns01) 'cannot handle instances without variables' snopt.(cns01,sl4cns01) 'fails due to initial singular point' snopt.(cns12,sl4cns12) 'fails due to badly scaled point' baron.(lp03,sl4lp03) 'I guess that BARONs presolve takes out the xx and does not postsolve the dual solution from the LP solver correctly; this was a temporary skip with "fix in progress" since the initial version of this file in 2007; moved it to permanent skips in 2020' (ipopt,ipopth).(lp10,sl4lp10) 'hits timelimit' snopt.scen03 'calls scensol1 which needs bratio=1 for snopt in order to solve correctly' $if %system.buildcode% == WEI mosek.(scensol6,sl4lp01,sl4lp02,sl4lp03,sl4lp10,sl4lp11) 'solvelink=4 gets stuck or fails with status=3/6, #3811' $if %system.buildcode% == LAG mosek.(qcp01,nlpqcp01,sl4qcp01) 'terminates due to slow progress' conopt3.conopt02 'test targets conopt4 multi-threading' alphaecp.miqcp04 'iffy solvestat/modelstat, #3583, will not fix' alphaecp.(sosmiqcp01,sosminlp01) 'wrong model status, #3621, will not fix' knitro.(mpsge01*mpsge02,mpsge10*mpsge14) 'knitro does not support MPSGE' knitro.(mcp01*mcp04,sl4mcp01*sl4mcp04) 'knitro cannot flag redef / modify duals for complementarity constraints with fixed variables as they get removed' knitro.(cns12,sl4cns12) 'knitro has weaker abs feasibility tolerance than required to report correct numinfeas' knitro.mcp11 'knitro does not solve up to required (unscaled) accuracy' (gurobi,gurobiold).(circlen,lonlp,nlp01,nlpqcp01,nlpqcp02) 'gurobi can only handle certain nonlinear constraints' (xpress,xpressold).(cns01,sl4cns01) 'xpress knitro falsely detects problem as infeasible' reshop.(emp05,emp18*emp26,emp28*emp32,emp34*emp35) 'JAMS specific tests' reshop.(empbp04,empecs02,empvi01) 'More JAMS specific tests' reshop.(emp27,empdisj1*empdisj7,emplog1*emplog2) 'Disjunctions/LogMIP are outside of the current scope of ReSHOP' reshop.(emppy1*emppy3) 'Test Python processor for empinfo' reshop.(empsp01) 'EMP/SP is outside the current scope of ReSHOP' $if %system.buildcode% == WEI (gurobi,gurobiold).(qcp02,sl4qcp02) 'Numerical issue' $if %system.buildcode% == DAC (gurobi,gurobiold).(qcp02,sl4qcp02) 'Numerical issue' * put temporary skips here (baron,lindoglobal,lindo).(qcp07) 'BARON and Lindo do not handle poly function' knitro.lp15 'knitro is far off the optimal value, #2346 and #2335' highs.mip05 'very slow' minos.(cns14,sl4cns14) 'strange convergence failure: gitlab issue #6464' snopt.sl4cns14 'strange convergence failure: gitlab issue #6466' (xpress,xpressold).(cns14,sl4cns14) 'strange convergence failure: gitlab issue #6468' reshop.(emp04) 'Support for "keepobj" JAMS option beyond GAMS 48' reshop.(emp06*emp08,empecs01) 'dualvar not yet supported by ReSHOP' reshop.(emp12*emp16) 'JAMS test: uses jams option file for custom empinfo name' reshop.(empadj01) 'Support planned beyond GAMS 47' reshop.(empvi02*empvi04) 'Not all equations are assigned' reshop.(emp33) 'KKT system of given model has multiple solutions' reshop.(empbp06) 'ReSHOP does not support "phantom" variables yet' reshop.(emp09,empvi05) 'investigate starting values in ReSHOP' reshop.(emp17) 'ReSHOP needs fixing' /; * if a model is skipped for a solver alias it is automatically skipped for the solver itself skip(solver,'indic01')$[(not sameas(solver,'cplex')) and (not sameas(solver,'xpress')) and (not sameas(solver,'xpressold')) and (not sameas(solver,'scip')) and (not sameas(solver,'copt')) and (not sameas(solver,'gurobi'))] = yes; skip(solver,'indic02')$[(not sameas(solver,'cplex')) and (not sameas(solver,'xpress')) and (not sameas(solver,'xpressold')) and (not sameas(solver,'scip')) and (not sameas(solver,'copt')) and (not sameas(solver,'gurobi'))] = yes; skip(solver,'indic03')$[(not sameas(solver,'cplex')) and (not sameas(solver,'xpress')) and (not sameas(solver,'xpressold')) and (not sameas(solver,'scip')) and (not sameas(solver,'copt')) and (not sameas(solver,'gurobi'))] = yes; skip(solver,'indic04')$[(not sameas(solver,'cplex')) and (not sameas(solver,'xpress')) and (not sameas(solver,'xpressold')) and (not sameas(solver,'scip')) and (not sameas(solver,'copt')) and (not sameas(solver,'gurobi'))] = yes; skip(solver,'lazy01')$[(not sameas(solver,'gurobi')) and (not sameas(solver,'gurobiold'))] = yes; skip(solver,'sens01')$[(not sameas(solver,'gurobi')) and (not sameas(solver,'gurobiold')) and (not sameas(solver,'cplex')) and (not sameas(solver,'highs'))] = yes; skip(solver,'mipstopexpr1')$[(not sameas(solver,'gurobi')) and (not sameas(solver,'gurobiold')) and (not sameas(solver,'cplex')) and (not sameas(solver,'xpress')) and (not sameas(solver,'xpressold'))] = yes; skip(solver,'echoindic')$[(not sameas(solver,'cplex')) and (not sameas(solver,'xpress')) and (not sameas(solver,'xpressold')) and (not sameas(solver,'copt')) and (not sameas(solver,'gurobi'))] = yes; // scip has its own option reader, so it does not understand "echo off" skip('nlpec',mdl)$[sum{ms(mdl,'mpsge'),1}] = yes; $if set NOSKIP skip(solver,mdl) = no; set mdlstdoutskip(m) 'skips for stdout check' / $if not %system.filesys% == UNIX asynexec 'when killing a process, taskkill writes to stdout' asynntrp 'sometimes a process to be killed does not exist anymore, which results in a message to stdOut' call1,call2 'tests handling of wrong parameters, error is written to stdout' call3 'tests output conflicts, so we know it writes to sterr and stdout' cmexrc01 'we are forcing a CMEX error resulting in a message to stdout' cpplib00 'compilers write to both stderr and stdout by default' decis01*decis03 'decis writes to stdout' genfile01 'the test triggers terminate writing to stdOut' maxima 'creates a wanted Cmex error writing to stdout' pgams02 'writes message when deleting directory below 225' testexeq 'compiler write to both stderr and stdout by default' trilib01*trilib04 'unzip and compiler write to stdout and stderr by default' veda1 'tests gdx2veda which writes to stdout' kestrel1*kestrel5 'the kestreltest script writes its log to stdout' scensol6 'deletion of grid files is reported to stdout' doc01 'documentation tests write to stdout' procdir1 'model uses keep option, which writes to stdout' emppy1*emppy3 'Python package ply write sometimes "Generating LALR tables" to stdout' mipstopexpr1 'checks mip stop expression compilation error which writes to stdout/stderr' rs02 'gurobi 11 writes to stdout' /; set mdlstderrskip(m) 'skips for stderr check' / call3 'tests output conflicts, so we know it writes to sterr and stdout' cpplib00 'compilers write to both stderr and stdout by default' decis01*decis03 'decis writes to stderr' testexeq 'compiler write to both stderr and stdout by default' trilib01*trilib04 'unzip and compiler write to stdout and stderr by default' doc01 'documentation tests may write to stderr' mipstopexpr1 'checks mip stop expression compilation error which writes to stdout/stderr' $if %system.buildcode% == DEG optdir1 'shot cannot turn off CPLEX output completely' /; set slvstdoutskip(solver) 'solvers which are skipped for stdout check' / $if %system.buildcode% == DEG shot 'cannot turn off CPLEX output completely' /; set slvstderrskip(solver) 'solvers which are skipped for stderr check' / $if %system.buildcode% == DEG shot 'cannot turn off CPLEX output completely' /; set stdoutskip(solver,m) 'model - solver combinations which are skipped for stdout check' / /; set stderrskip(solver,m) 'model - solver combinations which are skipped for stderr check' / /; $offEmpty stdoutskip(slvstdoutskip,mdl)$(not mt(mdl,'GAMS')) = yes; stdoutskip(solver,mdlstdoutskip)$(not mt(mdlstdoutskip,'GAMS')) = yes; stderrskip(slvstderrskip,mdl)$(not mt(mdl,'GAMS')) = yes; stderrskip(solver,mdlstderrskip)$(not mt(mdlstderrskip,'GAMS')) = yes; $onEmpty set mdltraceskip(m) 'skips for trace file check' / badpt2,badpt3 '9/14 is expected outcome of this test' qcp06 '2nd solve is supposed to fail' scensol3 '%solveStat.terminatedBySolver% is expected' /; set slvtraceskip(solver) 'solvers which are skipped for trace file check' / /; set traceskip(solver,m) 'model - solver combinations which are skipped for trace file check' / /; $offEmpty $if exist %sysenv.USRQUALITYSKIPS% $include %sysenv.USRQUALITYSKIPS% traceskip(slvtraceskip,mdl)$(not mt(mdl,'GAMS')) = yes; traceskip(solver,mdltraceskip)$(not mt(mdltraceskip,'GAMS')) = yes; * Solver test suites set slvtest(solver,suite) / baron.baron (conopt3,conopt).conopt convert.convert copt.copt cplex.cplex jams.emp examiner.examiner gamschk.gamschk gurobi.gurobi highs.highs knitro.knitro (lindo,lindoglobal).lindoglobal miles.miles minos.minos mosek.mosek nlpec.nlpec path.path pathnlp.pathnlp reshop.(emp,reshop) sbb.sbb scip.scip (cbc,conopt,conopt3,cplex,gurobi,gurobiold,ipopt,ipopth,lindo,lindoglobal,minos,mosek,odhcplex,soplex,path,scip,snopt,xpress,xpressold).guss (xpress,xpressold).xpress /; loop(ASM(solverAlias,solver), slvtest(solver,suite) $= slvtest(solverAlias,suite); slvtest(solverAlias,suite) $= slvtest(solver,suite); ); alias(solver,solverp); set slvsuite(suite); option slvsuite < slvtest; SET moreskip(solver,m) 'more skips, based on solver suites to use'; * start by skipping all models in the solver suites moreskip(solver,mdl)$sum(ms(mdl,slvsuite),1) = yes; * but include the models in solver suites chosen to run, * if the solver is available moreskip(solver,mdl)$sum((slvtest(avail(solver),slvsuite),ms(mdl,slvsuite)),1) = no; * display moreskip; skip(moreskip(solver,mdl)) = yes; skip(suiteskip(solver,mdl)) = yes; $if %DEMOSIZE% == 1 skip(solver, mdl)$sum(sm(s,mdl)$(big(s)), 1) = yes; $if %DEMOSIZE% == 1 skip(globalSolver,mdl)$sum(sm(s,mdl)$(bigGlobal(s)),1) = yes; gskip(mdl)$[sum{suiteskip(solver,mdl),1}] = yes; gskip(mdl)$[sum((slvtest(solver,slvsuite),ms(mdl,slvsuite)), not avail(solver)) ] = yes; $if %DEMOSIZE% == 1 gskip(mdl)$sum(sm(s,mdl)$(big(s)),1) = yes; $if set QUICK skip(solver,mdl)$notquick(mdl) = yes; $if set GAZ skip(solver,mdl)$notGAZ(mdl) = yes; * display gskip, skip; file log /''/; $if not set HELP $goTo nohelp put log / 'This model is used to run the quality tests and report on the results' / 'You can run all tests for all solvers by running the model with no parameters:' / ' ' / ' gams quality' / ' ' / 'You can choose specific tests by using the --TEST parameter:' / ' ' / ' gams quality --TEST "binary1,binary2"' / ' ' / 'Instead of choosing specific tests, you can choose' / 'specific test suites by using the --SUITE parameter:' / ' ' / ' gams quality --SUITE "cmex01,lp"' / ' ' / 'Independently from the parameters above, you can make' / 'a choice of solvers using the --SOLVER parameter:' / ' ' / ' gams quality --SUITE lp --SOLVER "soplex,highs"' / ' ' / 'By default, a skip list is used to skip tests that are known to fail.' / 'To turn off the skip list and really run all the tests,' / 'set the --NOSKIP parameter to anything:' / ' ' / ' gams quality --NOSKIP=1' / ' ' / 'By default, a solver with one or more aliases is only tested once. To run the' / 'tests with the aliased names as well set the --RUNALIAS parameter to anything:' / ' ' / ' gams quality --RUNALIAS=1' / ' ' / 'You can knock out the slower tests (see the set notQuick)' / 'using the --QUICK parameter:' / ' ' / ' gams quality --QUICK=1' / ' ' / 'Internal use: you can run a quick test requiring no license and' / 'avoiding things not provided by a gaz build (see the set notGAZ)' / 'using the --GAZ parameter:' / ' ' / ' gams quality --GAZ=1' / ' ' / 'To run only those tests that fit within the demo size limits, do:' / ' ' / ' gams quality --DEMOSIZE=1' / ' '; $exit $label nohelp SET tmdl(m) 'set of tests to run'; SET tsuite(suite) 'set of suites to run'; SET tsolver(solver) 'set of solvers to test'; $if not errorfree $abort Compilation errors before processing -- options $if not set TEST $goTo checksuite SET tmdl(m) / %TEST% /; $if errorfree $goTo TEST_OK $clearError $log The test model(s) specified (--TEST=%TEST%) are not all valid $abort $label TEST_OK tsuite(suite) = no; $goTo havetestlist $label checksuite $if not set SUITE $goTo alltests SET tsuite(suite) / %SUITE% /; $if errorfree $goTo SUITE_OK $clearError $log The suite(s) specified (--SUITE=%SUITE%) are not all valid $abort $label SUITE_OK tmdl(mdl) = sum{ms(mdl,tsuite),yes}; $goTo havetestlist $label alltests tsuite(suite) = no; tmdl(mdl) = yes; $label havetestlist display tmdl; $if not set SOLVER $goTo allsolvers SET tsolver(solver) / %SOLVER% /; gskip(mdl) = yes; gskip(mdl)$[sum((slvtest(tsolver,slvsuite),ms(mdl,slvsuite)),avail(tsolver)) ] = no; $if errorfree $goTo SOLVER_OK $clearError $log The solver(s) specified (--SOLVER=%SOLVER%) are not all valid $abort $label SOLVER_OK $goTo havesolvers $label allsolvers tsolver(solver) = yes; tsolver(filter(solver)) = no; $if set RUNALIAS $goTo havesolvers tsolver(solverAlias(solver)) = no; $label havesolvers loop(ASM(solverAlias,solver), solverFlags(solver) $= solverFlags(solverAlias); modelSolverFlags(mdl,solver) $= modelSolverFlags(mdl,solverAlias); stdoutskip(solver,mdl) $= stdoutskip(solverAlias,mdl); stderrskip(solver,mdl) $= stderrskip(solverAlias,mdl); skip(solver,mdl) $= skip(solverAlias,mdl); solverFlags(solverAlias) $= solverFlags(solver); modelSolverFlags(mdl,solverAlias) $= modelSolverFlags(mdl,solver); stdoutskip(solverAlias,mdl) $= stdoutskip(solver,mdl); stderrskip(solverAlias,mdl) $= stderrskip(solver,mdl); skip(solverAlias,mdl) $= skip(solver,mdl); ); scalar rc / 0 /, tot / 0 /, err / 0 /, stderr / 0 /, stdout / 0 /, cnt / 0 /; file oneTest / 'onetest.gms' / //Getting test from testlib and executing test stdTest / 'stdtest.gms' / //Checking if stderr and stdout are empty allTests / '%ALL%' / failures / '%FAIL%' / rmme / 'rmme.gms' / fx; oneTest.tf = 0; oneTest.lcase = 1; $if %ALL% == alltests.gms putclose allTests '* These are the tests we ran' /; $if %FAIL% == failures_qa.gms putclose failures '* These are the tests that failed' / '*Total tests: 0 Failed tests: 0' /; putclose rmme '* Delete all directories of tests without problems' /; allTests.ap = 1; failures.ap = 1; rmme.ap = 1; $if exist %TRACE% $call rm -f %TRACE% $set DIRNAMEGAMS "'%PREFIX%_qa_' tmdl.tl:0 '_gams'" loop {mt(tmdl,'GAMS')$[not (gskip(tmdl) or ms(tmdl,'refact'))], cnt = cnt + 1; tot = tot + 1; put_utility fx 'shell' / 'rm -rf ' %DIRNAMEGAMS%; put_utility fx 'shell' / 'mkdir ' %DIRNAMEGAMS%; put oneTest '$call testlib -q ' tmdl.tl:0 / '$if errorlevel 1 $abort' / '$echo JobStart ' tmdl.tl:0 ' >> %TRACE%' / '$call =gams ' tmdl.tl:0 ' jt=' tmdl.tl:0 ' trace=%TRACE% reslim=1000 ' modelFlags.te(tmdl):0 ' %gams.user1% %FLAGS%'; put$(not mdltraceskip(tmdl)) / '$call gams %TRACE% a=gt ps=0 pw=255 tl=0 lo=0'; putclose / '$if errorlevel 1 $set err 1' / '$if set err scalar err /%err%/' / '$call cat %TRACE% >> ..%SLASH%%TRACE%' / "$if %err% == 1 $abort 'Problem'"; put_utility fx 'shell' / 'mv -f onetest.gms ' %DIRNAMEGAMS% ' && cd ' %DIRNAMEGAMS% ' && gams onetest lo=%GAMSlo% --err=0 gdx=..%SLASH%err > stdout.txt 2>stderr.txt'; rc=1; execute_load 'err' rc=err; putclose stdTest //Checking if stderr and stdout are empty '$offCheckErrorLevel' / '$call =test -s ' %DIRNAMEGAMS% '%SLASH%stderr.txt' / '$if errorlevel 1 $set stderr 0' / '$if set stderr scalar stderr /%stderr%/' / '$call =test -s ' %DIRNAMEGAMS% '%SLASH%stdout.txt' / '$if errorlevel 1 $set stdout 0' / '$if set stdout scalar stdout /%stdout%/' execute 'gams stdtest.gms lo=2 --stderr=1 --stdout=1 gdx=std'; stderr=1; stdout=1; execute_load 'std' stderr,stdout; put allTests; put '$call =gams quality %FLAGS% --prefix=%PREFIX% --fail=failures_qa.tmp --test=' tmdl.tl:0 ' '; if {rc, put '--ftrace=1 '}; if {stderr and not mdlstderrskip(tmdl), put '--fstderr=1 '}; if {stdout and not mdlstdoutskip(tmdl), put '--fstdout=1 '}; putclose allTests '--dir='%DIRNAMEGAMS% /; if {rc or (stderr and not mdlstderrskip(tmdl)) or (stdout and not mdlstdoutskip(tmdl)), err = err + 1; put failures; // failure_qa.gms put '$call =gams quality %FLAGS% --prefix=%PREFIX% --fail=failures_qa.tmp --test=' tmdl.tl:0 ' '; // --fail defines alternative failures file if {rc, put '--ftrace=1 '}; // execution error if {stderr and not mdlstderrskip(tmdl), put '--fstderr=1 '}; // writing to standard error if {stdout and not mdlstdoutskip(tmdl), put '--fstdout=1 '}; // writing to standard output putclose failures '--dir='%DIRNAMEGAMS% /; // name of kept directory else putclose rmme '$call rm -rf ' %DIRNAMEGAMS% /; }; if(cnt=5, execute "=gams rmme lo=0"; cnt = 0; execute "echo '* Delete all directories of tests without problems' > rmme.gms"; ); }; $set DIRNAMEREFACT "'%PREFIX%_qa_refact'" if(tsuite('refact'), put_utility fx 'shell' / 'if not exist ' %DIRNAMEREFACT% ' mkdir ' %DIRNAMEREFACT%; ) loop {ms(tmdl,'refact')$tsuite('refact'), cnt = cnt + 1; tot = tot + 1; put oneTest '$if exist %TRACE% $call rm -f %TRACE%' / '$call testlib -q ' tmdl.tl:0 / '$if errorlevel 1 $abort' / '$echo JobStart ' tmdl.tl:0 ' >> %TRACE%' / '$call =gams ' tmdl.tl:0 ' jt=' tmdl.tl:0 ' trace=%TRACE% reslim=1000 ' modelFlags.te(tmdl):0 ' %gams.user1% %FLAGS%'; put$(not mdltraceskip(tmdl)) / '$call gams %TRACE% a=gt ps=0 pw=255 tl=0 lo=0'; putclose / '$if errorlevel 1 $set err 1' / '$if set err scalar err /%err%/' / '$call cat %TRACE% >> ..%SLASH%%TRACE%' / "$if %err% == 1 $abort 'Problem'"; put_utility fx 'shell' / 'mv -f onetest.gms ' %DIRNAMEREFACT% ' && cd ' %DIRNAMEREFACT% ' && gams onetest lo=%GAMSlo% --err=0 gdx=..%SLASH%err > stdout.txt 2>stderr.txt'; rc=1; execute_load 'err' rc=err; putclose stdTest //Checking if stderr and stdout are empty '$offCheckErrorLevel' / '$call =test -s ' %DIRNAMEREFACT% '%SLASH%stderr.txt' / '$if errorlevel 1 $set stderr 0' / '$if set stderr scalar stderr /%stderr%/' / '$call =test -s ' %DIRNAMEREFACT% '%SLASH%stdout.txt' / '$if errorlevel 1 $set stdout 0' / '$if set stdout scalar stdout /%stdout%/' execute 'gams stdtest.gms lo=2 --stderr=1 --stdout=1 gdx=std'; stderr=1; stdout=1; execute_load 'std' stderr,stdout; put allTests; put '$call =gams quality %FLAGS% --prefix=%PREFIX% --fail=failures_qa.tmp --test=' tmdl.tl:0 ' '; if {rc, put '--ftrace=1 '}; if {stderr and not mdlstderrskip(tmdl), put '--fstderr=1 '}; if {stdout and not mdlstdoutskip(tmdl), put '--fstdout=1 '}; putclose allTests '--dir='%DIRNAMEREFACT% /; if {rc or (stderr and not mdlstderrskip(tmdl)) or (stdout and not mdlstdoutskip(tmdl)), err = err + 1; put failures; // failure_qa.gms put '$call =gams quality %FLAGS% --prefix=%PREFIX% --fail=failures_qa.tmp --test=' tmdl.tl:0 ' '; // --fail defines alternative failures file if {rc, put '--ftrace=1 '}; // execution error if {stderr and not mdlstderrskip(tmdl), put '--fstderr=1 '}; // writing to standard error if {stdout and not mdlstdoutskip(tmdl), put '--fstdout=1 '}; // writing to standard output putclose failures '--dir='%DIRNAMEREFACT% /; // name of kept directory }; }; $onEcho > xpress.500 bargapstop 1e-8 barprimalstop 1e-7 bardualstop 1e-7 $offEcho $onEcho > xpressold.500 bargapstop 1e-8 barprimalstop 1e-7 bardualstop 1e-7 $offEcho $onEcho > cplex.100 rerun yes $offEcho $onEcho > gurobi.100 rerun 0 $offEcho $onEcho > gurobiold.100 rerun 0 $offEcho $onEcho > scip.100 nlpi/ipopt/priority = 3000 $offEcho $set DIRNAME "'%PREFIX%_qa_' tmdl.tl:0 '_' type.tl:0 '_' tsolver.tl:0" loop {mt(tmdl,type)$[not sameas(type,'GAMS')], loop {tsolver$[SolverCapabilities(tsolver,type) and avail(tsolver) and not skip(tsolver,tmdl)], cnt = cnt + 1; tot = tot + 1; put_utility fx 'shell' / 'rm -rf ' %DIRNAME%; put_utility fx 'shell' / 'mkdir ' %DIRNAME%; put oneTest '$call testlib -q ' tmdl.tl:0 / '$if errorlevel 1 $abort' / '$echo JobStart ' tmdl.tl:0 ' >> %TRACE%' / '$call =gams ' tmdl.tl:0 ' jt=' tmdl.tl:0 ' trace=%TRACE% reslim=1000 ' type.tl:0'='tsolver.tl:0 ' %FLAGS% ' solverFlags.te(tsolver):0 ' ' modelFlags.te(tmdl):0 ' ' modelSolverFlags.te(tmdl,tsolver):0 ' %gams.user1%'; put$(not traceskip(tsolver,tmdl)) / '$call gams %TRACE% a=gt ps=0 pw=255 tl=0 lo=0'; putclose / '$if errorlevel 1 $set err 1' / '$if set err scalar err /%err%/' / '$call cat %TRACE% >> ..%SLASH%%TRACE%' / "$if %err% == 1 $abort 'Problem'"; put_utility fx 'shell' / 'mv -f onetest.gms ' %DIRNAME% ' && cd ' %DIRNAME% ' && gams onetest lo=%GAMSlo% --err=0 gdx=..%SLASH%err > stdout.txt 2>stderr.txt'; rc=1; execute_load 'err' rc=err; putclose stdTest '$offCheckErrorLevel' / '$call =test -s ' %DIRNAME% '%SLASH%stderr.txt' / '$if errorlevel 1 $set stderr 0' / '$if set stderr scalar stderr /%stderr%/' / '$call =test -s ' %DIRNAME% '%SLASH%stdout.txt' / '$if errorlevel 1 $set stdout 0' / '$if set stdout scalar stdout /%stdout%/'; execute 'gams stdtest.gms lo=2 --stderr=1 --stdout=1 gdx=std'; stderr=1; stdout=1; execute_load 'std' stderr,stdout; put allTests; put '$call =gams quality %FLAGS% --prefix=%PREFIX% --fail=failures_qa.tmp --test=' tmdl.tl:0 ' --solver=' tsolver.tl:0 ' '; if {rc, put '--ftrace=1 '}; if {stderr and not stderrskip(tsolver,tmdl), put '--fstderr=1 '}; if {stdout and not stdoutskip(tsolver,tmdl), put '--fstdout=1 '}; putclose allTests '--dir='%DIRNAME% /; if {rc or (stderr and not stderrskip(tsolver,tmdl)) or (stdout and not stdoutskip(tsolver,tmdl)), err = err + 1; put failures; put '$call =gams quality %FLAGS% --prefix=%PREFIX% --fail=failures_qa.tmp --test=' tmdl.tl:0 ' --solver=' tsolver.tl:0 ' '; if {rc, put '--ftrace=1 '}; if {stderr and not stderrskip(tsolver,tmdl), put '--fstderr=1 '}; if {stdout and not stdoutskip(tsolver,tmdl), put '--fstdout=1 '}; putclose failures '--dir='%DIRNAME% /; else putclose rmme '$call rm -rf ' %DIRNAME% /; }; if(cnt=5, execute "=gams rmme lo=0"; cnt = 0; execute "echo '* Delete all directories of tests without problems' > rmme.gms"; ); }; }; execute "=gams %TRACE% a=gt ps=0 pw=255 o=%TRACEREP% tl=%TL% lo=0"; execute "=gams rmme lo=0"; putclose oneTest '*Total tests: ', tot:0:0, ' Failed tests: ', err:0:0 ; $if %FAIL% == failures_qa.gms execute 'cat onetest.gms >> %FAIL%' execute 'rm -f rmme.* err.gdx std.gdx onetest.gms stdtest.*'; put log; if {(err > 0), put 'There were errors: ', err:0:0, ' out of ', tot:0:0, ' tests failed.' /; put 'See the file failures_qa.gms to reproduce the failed runs'/; put 'You have some failures. See failures_qa.gms for details.'; else put 'Congratulations! All ', tot:0:0, ' tests passed.'/; }; put 'See the file alltests.gms to reproduce all the runs'/;