$title 'Test various GDX APIs and report run times' (gdxperf,SEQ=62)

$onText
Without any options this produces a GDX file from trnsport and exercises all GDX API. 
With --RUN=single --SINGLERUN=name.gdx this will do the same for any given GDX file.
With --RUNDEFAULT=0 one can turn off all API test and selectively turn them on with 
--RUNXYZ=1 (with XYZ being GAMS, G2NP, GTP, GMD, OOAPI, GDX, CTYPES, and CAPI).
The CAPI requires a C compiler to be installed. If the shared library can't be build, 
this API test is skipped even if requested.

The model can also work with subdirctories containing many GDX files. For this set
--RUN=suite and --SUITE=dirname or --SUITE=ALL to run a predefined set of suites:
  - lwsup
  - mrb
  - sqagams
  - src

The timing information can we written to the GAMS log (default), stdout (--LOGFILE=stdout)
or any file (--LOGFILE=name.txt).

Contributor: Michael Bussieck, December 2022
$offText


$if not set RUN $set RUN ELSE_CASE
$ifThenI %RUN%==SINGLE
$if not set SINGLERUN $set SINGLERUN ./sqagams/ELMOD.gdx
set fgdx / '%SINGLERUN%' /;

$elseIfI %RUN%==SUITE
set suites /
             lwproj  "Collection of 499 GDX found on Lutz Westermanns drive related to projects"
             lwsup   "Collection of 303 GDX found on Lutz Westermanns drive resulting from [personal] support"
             mrb     "Collection of  73 GDX found on Michael Bussiecks drive (often resulting from [personal] support)"
             sqagams "Collection of 534 GDX Files from sqagams excluding gamsmatr*.gdx"
             src     "Collection of 177 GDX found in products/src"
           /;
$onEmbedded
set drop(suites,*) /
          lwproj.(('rank_out.gdx','gmsgrid.gdx')                         "https://git.gams.com/devel/products/-/issues/5412"
                   'validnlpecopts.gdx'                                  "https://git.gams.com/devel/products/-/issues/5413"
                   'sol.gdx'                                             "https://git.gams.com/devel/products/-/issues/5419"
                  ('Test_Leitung2_out.gdx', 'Test_Leitung2_visual.gdx',
                   'Test_Leitung_out.gdx', 'Test_Leitung_visual.gdx')    "https://git.gams.com/devel/products/-/issues/5440"  )
           lwsup.(('5062.gdx', 'data1000.gdx', 'data10000.gdx'
                   'Modellendogen0004.gdx', 'Modellendogen0005.gdx', 'Modellendogen0725.gdx', 'Modellendogen0726.gdx', 'Modellendogen1093.gdx', 'Modellendogen1407.gdx'
                   'output2.gdx', 'output_DisEmp.gdx'
                   'preFix.gdx'
                   'soln_Toroptimierung_2_p1.gdx', 'Toroptimierung_2_p.gdx', 'Toroptimierung_2_p - Copy.gdx'
                   'z.gdx')                                                    "https://git.gams.com/devel/products/-/issues/5440"
                   'data_m5.gdx'                                               "https://git.gams.com/devel/products/-/issues/5419"
                  ('dbX2370.gdx', 'dbX2372.gdx')                               "https://git.gams.com/devel/products/-/issues/5439"
                   'gmsgrid.gdx'                                               "https://git.gams.com/devel/products/-/issues/5412"
                  ('testInstance1000Ite.gdx', 'testInstance100Ite.gdx', 'testInstance200Ite.gdx'
                   'x0.gdx','x1.gdx','x170.gdx','x170postfix.gdx','x171.gdx')  "https://git.gams.com/devel/products/-/issues/5441")
             mrb.('test.gdx','gdxdump.gdx',
                  'map.gdx', 'new4.gdx'             "https://git.gams.com/devel/products/-/issues/5419"
                  'food_man.gdx'                    "https://git.gams.com/devel/gams-transfer-python/-/issues/26"
                  'testExcel.gdx'                   "big, requires 25GB for g2np")
         sqagams.('ELMOD.gdx','JMM.gdx',
                  'arauco.gdx',
                  'modelo.gdx',
                  'PERSEUS.gdx'                     "big, requires 25GB for g2np"
                  'AssignmentBug.gdx','globiom.gdx',
                  'indata.gdx', 'seders.gdx'        "https://git.gams.com/devel/products/-/issues/5413"
                  'CHPSystemData2_Converted.gdx'    "https://git.gams.com/devel/gams-transfer-python/-/issues/27"
                  'pegase.gdx'                      "large scalar model, a killer for gtp")
             src.('getdata.gdx'                     "https://git.gams.com/devel/gams-transfer-python/-/issues/26"
                  'rank_out.gdx', 'rank_output.gdx',
                  'bau_p.gdx'                       "https://git.gams.com/devel/products/-/issues/5412"
                  'validnlpecopts.gdx','xyz.gdx'    "https://git.gams.com/devel/products/-/issues/5413")
           /;
$offEmbedded
$if not set SUITE $set SUITE ALL
set suite(*) /
$ifI not x%SUITE%==xALL %SUITE%
$ifI     x%SUITE%==xALL #suites
/;

set fgdx(*) "GDX files to run";
$onEmbeddedCode Python:
import glob
import os
f = []
suite = list(gams.get('suite'))
for s in suite:
   f = f + glob.glob(f'./{s}/*.gdx')
for d in gams.get('drop'):
   if d[0] in suite:
      # print(f'./{d[0]}{os.sep}{d[1]}')
      f.remove(f'./{d[0]}{os.sep}{d[1]}')

# Use the code to skip all models in the list up and including the model given by the name
#for i in range(f.index('./lwproj\\Test_Leitung2_out.gdx')+1): f.pop(0)
gams.set('fgdx',f)
$offEmbeddedCode fgdx

$elseIfI %RUN%==HARDCODE
* Some hand coded GDX file list
$inlineCom /* */
set fgdx / '1.gdx', '2.gdx', '3.gdx' /
$else
* Get trnsport from model lib and run that
$call gamslib -q trnsport
$call gams trnsport a=c lo=0 gdx=trnsport.gdx
set fgdx / 'trnsport.gdx' /
$endIf

$if not set LOGFILE    $set LOGFILE    GAMSLOG
$if not set RUNDEFAULT $set RUNDEFAULT 1
$if not set RUNGAMS    $set RUNGAMS    %RUNDEFAULT%
$if not set RUNG2NP    $set RUNG2NP    %RUNDEFAULT%
$if not set RUNGTP     $set RUNGTP     %RUNDEFAULT%
$if not set RUNGMD     $set RUNGMD     %RUNDEFAULT%
$if not set RUNOOAPI   $set RUNOOAPI   %RUNDEFAULT%
$if not set RUNGDX     $set RUNGDX     %RUNDEFAULT%
$if not set RUNCTYPES  $set RUNCTYPES  %RUNDEFAULT%
$if not set RUNCAPI    $set RUNCAPI    %RUNDEFAULT%

$onEmbeddedCode Python:
import time
import os
import sys
from gams.core.numpy import Gams2Numpy
import gams.transfer as gt
from gams.core.gdx import *
from gams.core.gmd import *
import numpy as np

def register_uels(gdx, uel_list, reverse=False):
   if reverse:
      uels = reversed(list(enumerate(uel_list[1:])))
   else:
      uels = list(enumerate(uel_list[1:]))

   rc = gdxUELRegisterMapStart(gdx)
   assert rc == 1
   for unr, u in uels:
       rc = gdxUELRegisterMap(gdx, unr+1, u)
       assert rc == 1
   rc = gdxUELRegisterDone(gdx)

step_cnt = 0
lastfn = ''
lastapi = ''

def printT(f, t, fn, api, text):
   global lastfn, lastapi, step_cnt 
   item_t = time.time() - t[2]
   total_t = t[0] + item_t 
   subtotal_t = t[1] + item_t
   if lastapi == api and fn == lastfn:
      step_cnt += 1
   else: 
      step_cnt = 0
      lastfn = fn
      lastapi = api
   if f == None:
       gams.printLog(f'{total_t:9.3f} # {subtotal_t:9.3f} # {item_t:9.3f} # {fn:10s} # {api} # {step_cnt} # {text}')
   else:
       f.write(f'{total_t:9.3f} # {subtotal_t:9.3f} # {item_t:9.3f} # {fn:10s} # {api} # {step_cnt} # {text}\n')
       f.flush()
   return [total_t, subtotal_t, time.time()]
t = [0,0,0]

if '%LOGFILE%' == 'stdout':
   flog = sys.stdout
elif '%LOGFILE%' == 'GAMSLOG':
   flog = None
else:
   flog = open('%LOGFILE%', 'w')
$offEmbeddedCode

* GAMS Test
$ifThen %RUNGAMS% == 1
$onEmbeddedCode Python:
for f in gams.get('fgdx'):
   t[1] = 0
   t[2] = time.time()
   with open('gdxdump.gms', "w") as fp:
      fp.write('$onUNDF\n')
   rc = os.system(f'gdxdump "{f}" noData >> gdxdump.gms')
   t = printT(flog, t, f, 'gams', 'gdxdump')
   assert rc == 0
   rc = os.system(f'gams gdxdump.gms lo=0')
   t = printT(flog, t, f, 'gams', 'read data - $load - gdxDataReadFilteredStart/gdxDataReadMap')
   assert rc == 0
   os.system(f'gams gdxdump.gms lo=0 gdx=default')
   t = printT(flog, t, f, 'gams', 'read and write data - dumpGDX - gdxDataWriteRaw')
   assert rc == 0
$offEmbeddedCode
$endIf

* GAMS2Numpy Test
$ifThen %RUNG2NP% == 1
$onEmbeddedCode Python:
import copy

system_directory = r'%gams.sysdir% '.rstrip()
g2np = Gams2Numpy(system_directory)
gdxHandle = new_gdxHandle_tp()
rc, msg = gdxCreateD(gdxHandle, system_directory, GMS_SSSIZE)
if not rc:
    raise Exception(msg)
wgdxHandle = new_gdxHandle_tp()
rc, msg = gdxCreateD(wgdxHandle, system_directory, GMS_SSSIZE)
if not rc:
    raise Exception(msg)

for f in gams.get('fgdx'):
   t[1] = 0
   t[2] = time.time()
   rc = gdxOpenRead(gdxHandle, f'{f}')[0]
   t = printT(flog, t, f, 'g2np', 'gdxOpenRead')
   assert rc == 1
   rc, symCount, _ = gdxSystemInfo(gdxHandle)
   t = printT(flog, t, f, 'g2np', f'handling {symCount} symbols')
   assert rc == 1

   for i in range(1, symCount + 1):
       rc, syid, sydim, sytype = gdxSymbolInfo(gdxHandle, i)
       if sytype == GMS_DT_ALIAS:
          continue
       keys, values = g2np.gdxReadSymbolRaw(gdxHandle, syid)
   t = printT(flog, t, f, 'g2np', 'reading symbols raw')

   uel_list = g2np.gdxGetUelList(gdxHandle)
   for i in range(1, symCount + 1):
       rc, syid, sydim, sytype = gdxSymbolInfo(gdxHandle, i)
       if sytype == GMS_DT_ALIAS:
          continue
       keys, values = g2np.gdxReadSymbolStr(gdxHandle, syid, uelList=uel_list)
   t = printT(flog, t, f, 'g2np', 'reading symbols str with uel list')

   for i in range(1, symCount + 1):
       rc, syid, sydim, sytype = gdxSymbolInfo(gdxHandle, i)
       if sytype == GMS_DT_ALIAS:
          continue
       keys, values = g2np.gdxReadSymbolStr(gdxHandle, syid)
   t = printT(flog, t, f, 'g2np', 'reading symbols str without uel list')

   # This GDX file won't have uels
   rc = gdxOpenWrite(wgdxHandle, "out.gdx", "g2np test")[0]
   for i in range(1, symCount + 1):
       rc, syid, sydim, sytype = gdxSymbolInfo(gdxHandle, i)
       if sytype == GMS_DT_ALIAS:
          continue
       keys, values = g2np.gdxReadSymbolRaw(gdxHandle, syid)
       g2np.gdxWriteSymbolRaw(wgdxHandle, syid, '', sydim, sytype, 0, keys, values)
   rc = gdxClose(wgdxHandle)
   t = printT(flog, t, f, 'g2np', 'reading and writing symbols raw')
   assert rc == 0

   rc = gdxOpenWrite(wgdxHandle, "out.gdx", "g2np test")[0]
   register_uels(wgdxHandle, uel_list, reverse=False)
   t = printT(flog, t, f, 'g2np', 'register uels (reverse=False)')

   for i in range(1, symCount + 1):
       rc, syid, sydim, sytype = gdxSymbolInfo(gdxHandle, i)
       if sytype == GMS_DT_ALIAS:
          continue
       keys, values = g2np.gdxReadSymbolStr(gdxHandle, syid, uelList=uel_list)
       g2np.gdxWriteSymbolStr(wgdxHandle, syid, '', sydim, sytype, 0, keys, values)
   rc = gdxClose(wgdxHandle)
   t = printT(flog, t, f, 'g2np', 'reading and writing symbols str reverse=False')
   assert rc == 0

   rc = gdxOpenWrite(wgdxHandle, "out.gdx", "g2np test")[0]
   register_uels(wgdxHandle, uel_list, reverse=True)
   t = printT(flog, t, f, 'g2np', 'register uels (reverse=True)')

   for i in range(1, symCount + 1):
       rc, syid, sydim, sytype = gdxSymbolInfo(gdxHandle, i)
       if sytype == GMS_DT_ALIAS:
          continue
       keys, values = g2np.gdxReadSymbolStr(gdxHandle, syid, uelList=uel_list)
       g2np.gdxWriteSymbolStr(wgdxHandle, syid, '', sydim, sytype, 0, keys, values)
   rc = gdxClose(wgdxHandle)
   t = printT(flog, t, f, 'g2np', 'reading and writing symbols str reverse=True')
   assert rc == 0

   rc = gdxClose(gdxHandle)
   assert rc == 0

gdxFree(wgdxHandle)
gdxFree(gdxHandle)
$offEmbeddedCode
$endIf

* GAMS Transfer Python Test
$ifThen %RUNGTP% == 1
$onEmbeddedCode Python:
import warnings
warnings.filterwarnings("ignore", category=UserWarning)

for f in gams.get('fgdx'):
   t[1] = 0
   t[2] = time.time() 
   m = gt.Container(f, system_directory=r"%gams.sysdir% ".strip())
   t = printT(flog, t, f, 'gtp', 'constructor with reading')

   m.write('out.gdx')
   t = printT(flog, t, f, 'gtp', 'write')
   
   # roll the last to the first to create out of order writing
   for sn,s in m.data.items():
      if not isinstance(s,gt.Alias) and not isinstance(s,gt.UniverseAlias) and s.number_records > 1:
         s.setRecords(s.records.apply(np.roll, shift=1))
   t = printT(flog, t, f, 'gtp', 'create out-of-order dataframes')
      
   m.write('out.gdx')
   t = printT(flog, t, f, 'gtp', 'write out-of-order')
$offEmbeddedCode
$endIf

* GMD Test
$ifThen %RUNGMD% == 1
$onEmbeddedCode Python:
system_directory = r'%gams.sysdir% '.rstrip()

for f in gams.get('fgdx'):
   t[1] = 0
   t[2] = time.time()
   gmdHandle = new_gmdHandle_tp()
   rc, msg = gmdCreateD(gmdHandle, system_directory, GMS_SSSIZE)
   if not rc:
       raise Exception(msg)
   rc = gmdInitFromGDX(gmdHandle, f'{f}')
   t = printT(flog, t, f, 'gmd', 'gmdInitFromGDX')
   assert rc == 1

   rc = gmdCloseGDX(gmdHandle, True)
   t = printT(flog, t, f, 'gmd', 'gmdCloseGDX')
   assert rc == 1

   rc = gmdWriteGDX(gmdHandle, 'out.gdx', False)
   t = printT(flog, t, f, 'gmd', 'gmdWriteGDX with domain checking')
   assert rc == 1

   rc = gmdWriteGDX(gmdHandle, 'out.gdx', True)
   t = printT(flog, t, f, 'gmd', 'gmdWriteGDX without domain checking')
   assert rc == 1

   gmdFree(gmdHandle)
$offEmbeddedCode
$endIf

* GAMS OO API
$ifThen %RUNOOAPI% == 1
$onEmbeddedCode Python:
from gams import *
system_directory = r'%gams.sysdir% '.rstrip()
ws = GamsWorkspace(system_directory=system_directory, working_directory='./')
for f in gams.get('fgdx'):
   t[1] = 0
   t[2] = time.time()
   db = ws.add_database_from_gdx(f'{f}')
   t = printT(flog, t, f, 'ooapi', 'ws.add_database_from_gdx')

   rc = gmdCloseGDX(db._gmd, True)
   t = printT(flog, t, f, 'ooapi', 'gmdCloseGDX')
   assert rc == 1

   db.export('out.gdx')
   t = printT(flog, t, f, 'ooapi', 'db.export with domain checking')

   db.suppress_auto_domain_checking = True
   db.export('out.gdx')
   t = printT(flog, t, f, 'ooapi', 'db.export without domain checking')
$offEmbeddedCode
$endIf

* GAMS GDX Python Test
$ifThen %RUNGDX% == 1
$onEmbeddedCode Python:
system_directory = r'%gams.sysdir% '.rstrip()
gdxHandle = new_gdxHandle_tp()
rc, msg = gdxCreateD(gdxHandle, system_directory, GMS_SSSIZE)
if not rc:
    raise Exception(msg)
wgdxHandle = new_gdxHandle_tp()
rc, msg = gdxCreateD(wgdxHandle, system_directory, GMS_SSSIZE)
if not rc:
    raise Exception(msg)
g2np = Gams2Numpy(system_directory)

for f in gams.get('fgdx'):
   t[1] = 0
   t[2] = time.time() 
   rc = gdxOpenRead(gdxHandle, f'{f}')[0]
   t = printT(flog, t, f, 'gdx', 'gdxOpenRead')
   assert rc == 1

   rc, symCount, _ = gdxSystemInfo(gdxHandle)
   t = printT(flog, t, f, 'gdx', f'handling {symCount} symbols')
   assert rc == 1

   for i in range(1, symCount + 1):
       rc, syid, sydim, sytype = gdxSymbolInfo(gdxHandle, i)
       rc, nrecs =  gdxDataReadRawStart(gdxHandle, i)
       assert rc == 1
       for k in range(nrecs):
           rc, elements, values, afdim = gdxDataReadRaw(gdxHandle)
           assert rc == 1
       gdxDataReadDone(gdxHandle)
   t = printT(flog, t, f, 'gdx', 'reading symbols raw')

   for i in range(1, symCount + 1):
       rc, syid, sydim, sytype = gdxSymbolInfo(gdxHandle, i)
       rc, nrecs =  gdxDataReadStrStart(gdxHandle, i)
       assert rc == 1
       for k in range(nrecs):
           rc, elements, values, afdim = gdxDataReadStr(gdxHandle)
           assert rc == 1
       gdxDataReadDone(gdxHandle)
   t = printT(flog, t, f, 'gdx', 'reading symbols str')
   
   uel_list = g2np.gdxGetUelList(gdxHandle)
   register_uels(gdxHandle, uel_list, reverse=False)
   t = printT(flog, t, f, 'gdx', 'register uels (reverse=False)')

   for i in range(1, symCount + 1):
       rc, syid, sydim, sytype = gdxSymbolInfo(gdxHandle, i)
       rc, nrecs =  gdxDataReadMapStart(gdxHandle, i)
       assert rc == 1
       for k in range(nrecs):
           rc, elements, values, afdim = gdxDataReadMap(gdxHandle,k+1)
           assert rc == 1
       gdxDataReadDone(gdxHandle)
   t = printT(flog, t, f, 'gdx', 'reading symbols map reverse=False')
   rc = gdxClose(gdxHandle)
   assert rc == 0

   rc = gdxOpenRead(gdxHandle, f'{f}')[0]
   t = printT(flog, t, f, 'gdx', 'gdxOpenRead')
   assert rc == 1
   
   register_uels(gdxHandle, uel_list, reverse=True)
   t = printT(flog, t, f, 'gdx', 'register uels (reverse=True)')

   for i in range(1, symCount + 1):
       rc, syid, sydim, sytype = gdxSymbolInfo(gdxHandle, i)
       rc, nrecs =  gdxDataReadMapStart(gdxHandle, i)
       assert rc == 1
       for k in range(nrecs):
           rc, elements, values, afdim = gdxDataReadMap(gdxHandle,k+1)
           assert rc == 1
       gdxDataReadDone(gdxHandle)
   t = printT(flog, t, f, 'gdx', 'reading symbols map reverse=True')

   rc = gdxOpenWrite(wgdxHandle, "out.gdx", "gdx test")[0]
   assert rc == 1
   for i in range(1, symCount + 1):
       rc, syid, sydim, sytype = gdxSymbolInfo(gdxHandle, i)
       assert rc == 1
       if sytype == GMS_DT_ALIAS:
          continue
       rc = gdxDataWriteRawStart(wgdxHandle, syid, "", sydim, sytype , 0)
       assert rc == 1
       rc, nrecs =  gdxDataReadRawStart(gdxHandle, i)
       assert rc == 1
       keys = intArray(sydim)
       vals = doubleArray(5)
       for j in range(nrecs):
           rc, elements, values, afdim = gdxDataReadRaw(gdxHandle)
           assert rc == 1
           # The copy should not be necessary!
           for k,v in enumerate(elements):
               keys[k] = v
           for k,v in enumerate(values):
               vals[k] = v
           rc = gdxDataWriteRaw(wgdxHandle, keys, vals)
           assert rc == 1
       gdxDataWriteDone(wgdxHandle)
       gdxDataReadDone(gdxHandle)
   rc = gdxClose(wgdxHandle)
   t = printT(flog, t, f, 'gdx', 'reading and writing symbols raw')
   assert rc == 0

   rc = gdxOpenWrite(wgdxHandle, "out.gdx", "gdx test")[0]
   register_uels(wgdxHandle, uel_list, reverse=False)
   t = printT(flog, t, f, 'gdx', 'register uels (reverse=False)')
   assert rc == 1

   for i in range(1, symCount + 1):
       rc, syid, sydim, sytype = gdxSymbolInfo(gdxHandle, i)
       assert rc == 1
       if sytype == GMS_DT_ALIAS:
          continue
       rc = gdxDataWriteMapStart(wgdxHandle, syid, "", sydim, sytype , 0)
       assert rc == 1
       rc, nrecs =  gdxDataReadRawStart(gdxHandle, i)
       assert rc == 1
       keys = intArray(sydim)
       vals = doubleArray(5)
       for j in range(nrecs):
           rc, elements, values, afdim = gdxDataReadRaw(gdxHandle)
           assert rc == 1
           # The copy should not be necessary!
           for k,v in enumerate(elements):
               keys[k] = v
           for k,v in enumerate(values):
               vals[k] = v
           rc = gdxDataWriteMap(wgdxHandle, keys, vals)
           assert rc == 1
       gdxDataWriteDone(wgdxHandle)
       gdxDataReadDone(gdxHandle)
   rc = gdxClose(wgdxHandle)
   t = printT(flog, t, f, 'gdx', 'reading raw and writing symbols map - reverse=False')
   assert rc == 0

   rc = gdxOpenWrite(wgdxHandle, "out.gdx", "gdx test")[0]
   register_uels(wgdxHandle, uel_list, reverse=False)
   t = printT(flog, t, f, 'gdx', 'register uels (reverse=True)')
   assert rc == 1

   for i in range(1, symCount + 1):
       rc, syid, sydim, sytype = gdxSymbolInfo(gdxHandle, i)
       assert rc == 1
       if sytype == GMS_DT_ALIAS:
          continue
       rc = gdxDataWriteMapStart(wgdxHandle, syid, "", sydim, sytype , 0)
       assert rc == 1
       rc, nrecs =  gdxDataReadRawStart(gdxHandle, i)
       assert rc == 1
       keys = intArray(sydim)
       vals = doubleArray(5)
       for j in range(nrecs):
           rc, elements, values, afdim = gdxDataReadRaw(gdxHandle)
           assert rc == 1
           # The copy should not be necessary!
           for k,v in enumerate(elements):
               keys[k] = v
           for k,v in enumerate(values):
               vals[k] = v
           rc = gdxDataWriteMap(wgdxHandle, keys, vals)
           assert rc == 1
       gdxDataWriteDone(wgdxHandle)
       gdxDataReadDone(gdxHandle)
   rc = gdxClose(wgdxHandle)
   t = printT(flog, t, f, 'gdx', 'reading raw and writing symbols map - reverse=True')
   assert rc == 0

   rc = gdxOpenWrite(wgdxHandle, "out.gdx", "gdx test")[0]
   register_uels(wgdxHandle, uel_list, reverse=False)
   t = printT(flog, t, f, 'gdx', 'register uels (reverse=False)')
   assert rc == 1

   for i in range(1, symCount + 1):
       rc, syid, sydim, sytype = gdxSymbolInfo(gdxHandle, i)
       assert rc == 1
       if sytype == GMS_DT_ALIAS:
          continue
       rc = gdxDataWriteStrStart(wgdxHandle, syid, "", sydim, sytype , 0)
       assert rc == 1
       rc, nrecs =  gdxDataReadStrStart(gdxHandle, i)
       assert rc == 1
       vals = doubleArray(5)
       for j in range(nrecs):
           rc, elements, values, afdim = gdxDataReadStr(gdxHandle)
           assert rc == 1
           # The copy should not be necessary!
           for k,v in enumerate(values):
               vals[k] = v
           rc = gdxDataWriteStr(wgdxHandle, elements, vals)
           assert rc == 1
       gdxDataWriteDone(wgdxHandle)
       gdxDataReadDone(gdxHandle)
   rc = gdxClose(wgdxHandle)
   t = printT(flog, t, f, 'gdx', 'reading and writing symbols str reverse-false')
   assert rc == 0

   rc = gdxOpenWrite(wgdxHandle, "out.gdx", "gdx test")[0]
   register_uels(wgdxHandle, uel_list, reverse=True)
   t = printT(flog, t, f, 'gdx', 'register uels (reverse=True)')
   assert rc == 1

   for i in range(1, symCount + 1):
       rc, syid, sydim, sytype = gdxSymbolInfo(gdxHandle, i)
       assert rc == 1
       if sytype == GMS_DT_ALIAS:
          continue
       rc = gdxDataWriteStrStart(wgdxHandle, syid, "", sydim, sytype , 0)
       assert rc == 1
       rc, nrecs =  gdxDataReadStrStart(gdxHandle, i)
       assert rc == 1
       vals = doubleArray(5)
       for j in range(nrecs):
           rc, elements, values, afdim = gdxDataReadStr(gdxHandle)
           assert rc == 1
           # The copy should not be necessary!
           for k,v in enumerate(values):
               vals[k] = v
           rc = gdxDataWriteStr(wgdxHandle, elements, vals)
       gdxDataWriteDone(wgdxHandle)
       gdxDataReadDone(gdxHandle)
   rc = gdxClose(wgdxHandle)
   t = printT(flog, t, f, 'gdx', 'reading and writing symbols str reverse-true')
   assert rc == 0

   rc = gdxClose(gdxHandle)
   assert rc == 0
   
$offEmbeddedCode
$endIf

* GAMS GDX Python Test
$ifThen %RUNCTYPES% == 1
$if not set GDXCCTYPE $set GDXCCTYPE cc
$ifThen.gdxccttype %GDXCCTYPE%==dc
$  set CGDX cgdx
$  set GDX gdx
$else.gdxccttype
$  set CGDX c__gdx
$  set GDX c__gdx
$endIf.gdxccttype

$onEmbeddedCode Python:
from ctypes import *

system_directory = r'%gams.sysdir% '.rstrip()
if sys.platform == "linux" or sys.platform == "linux2":
    so_name = "libgdx%GDXCCTYPE%lib64.so"
elif sys.platform == "darwin":
    so_name = "libgdx%GDXCCTYPE%lib64.dylib"
elif sys.platform == "win32":
    so_name = "gdx%GDXCCTYPE%lib64.dll"
else:
    raise Exception(f'unknown OS {sys.platform}')
gdx = cdll.LoadLibrary(os.path.join(system_directory, so_name))

gdxHandle = c_void_p(0)
gdx.xcreate(byref(gdxHandle))
wgdxHandle = c_void_p(0)
gdx.xcreate(byref(wgdxHandle))
ival = c_int(0)
ival_p = byref(ival)
jval = c_int(0)
sval = create_string_buffer(256)
uels = [create_string_buffer(64) for i in range(20)]
uelsptr = (c_char_p*20)(*map(addressof, uels))
vals = (c_double*5)((0.0)*5)
vals_p = pointer(vals)
rawuels = (c_int*20)((0)*20)
rawuels_p = pointer(rawuels)

for f in gams.get('fgdx'):
   t[1] = 0
   t[2] = time.time()
   rc = gdx.%CGDX%openread(gdxHandle, c_char_p(f.encode()), byref(ival))
   t = printT(flog, t, f, 'ctypes', 'cgdxopenread')
   assert rc == 1

   rc = gdx.%GDX%systeminfo(gdxHandle, byref(ival), byref(jval))
   symCount = ival.value;
   t = printT(flog, t, f, 'ctypes', f'handling {symCount} symbols')
   assert rc == 1

   for i in range(1, symCount + 1):
       rc = gdx.%CGDX%symbolinfo(gdxHandle, c_int(i), sval, byref(ival), byref(jval))
       syid = sval.value
       sydim = ival.value
       sytype = jval.value
       assert rc == 1
       if sytype == 4: # GMS_DT_ALIAS
          continue
       rc = gdx.%GDX%datareadrawstart(gdxHandle, c_int(i), byref(ival))
       nrecs = ival.value
       assert rc == 1
       for j in range(nrecs):
           rc = gdx.%GDX%datareadraw(gdxHandle, rawuels_p, vals_p, ival_p)
           assert rc == 1
       rc = gdx.%GDX%datareaddone(gdxHandle)
   t = printT(flog, t, f, 'ctypes', 'reading symbols raw')

   for i in range(1, symCount + 1):
       rc = gdx.%CGDX%symbolinfo(gdxHandle, c_int(i), sval, byref(ival), byref(jval))
       syid = sval.value
       sydim = ival.value
       sytype = jval.value
       assert rc == 1
       if sytype == 4: # GMS_DT_ALIAS
          continue
       rc = gdx.%GDX%datareadstrstart(gdxHandle, c_int(i), byref(ival))
       nrecs = ival.value
       assert rc == 1
       for j in range(nrecs):
           rc = gdx.%CGDX%datareadstr(gdxHandle, uelsptr, vals_p, ival_p)
           assert rc == 1
       rc = gdx.%GDX%datareaddone(gdxHandle)
   t = printT(flog, t, f, 'ctypes', 'reading symbols str')

   rc = gdx.%CGDX%openwrite(wgdxHandle, c_char_p(b'out.gdx'), c_char_p(b'ctypes experiment'), byref(ival))
   t = printT(flog, t, f, 'ctypes', 'cgdxopenwrite')
   assert rc == 1

   for i in range(1, symCount + 1):
       rc = gdx.%CGDX%symbolinfo(gdxHandle, c_int(i), sval, byref(ival), byref(jval))
       syid = sval.value
       sydim = ival.value
       sytype = jval.value
       assert rc == 1
       if sytype == 4: # GMS_DT_ALIAS
          continue
       rc = gdx.%CGDX%datawriterawstart(wgdxHandle, c_char_p(syid), c_char_p(b''), c_int(sydim), c_int(sytype), c_int(0))
       assert rc == 1
       rc = gdx.%GDX%datareadrawstart(gdxHandle, c_int(i), byref(ival))
       nrecs = ival.value
       assert rc == 1
       for j in range(nrecs):
           rc = gdx.%GDX%datareadraw(gdxHandle, rawuels_p, vals_p, ival_p)
           assert rc == 1
           rc = gdx.%GDX%datawriteraw(wgdxHandle, rawuels, vals)
           assert rc == 1
       rc = gdx.%GDX%datawritedone(wgdxHandle)
       rc = gdx.%GDX%datareaddone(gdxHandle)
   rc = gdx.%GDX%close(wgdxHandle)
   t = printT(flog, t, f, 'ctypes', 'reading and writing symbols raw')
   assert rc == 0

   rc = gdx.%CGDX%openwrite(wgdxHandle, c_char_p(b'out.gdx'), c_char_p(b'ctypes experiment'), byref(ival))
   t = printT(flog, t, f, 'ctypes', 'cgdxopenwrite')
   assert rc == 1

   for i in range(1, symCount + 1):
       rc = gdx.%CGDX%symbolinfo(gdxHandle, c_int(i), sval, byref(ival), byref(jval))
       syid = sval.value
       sydim = ival.value
       sytype = jval.value
       assert rc == 1
       if sytype == 4: # GMS_DT_ALIAS
          continue
       rc = gdx.%CGDX%datawritestrstart(wgdxHandle, c_char_p(syid), c_char_p(b''), c_int(sydim), c_int(sytype), c_int(0))
       assert rc == 1
       rc = gdx.%GDX%datareadstrstart(gdxHandle, c_int(i), byref(ival))
       nrecs = ival.value
       assert rc == 1
       for j in range(nrecs):
           rc = gdx.%CGDX%datareadstr(gdxHandle, uelsptr, vals_p, ival_p)
           assert rc == 1
           rc = gdx.%CGDX%datawritestr(wgdxHandle, uelsptr, vals)
           assert rc == 1
       rc = gdx.%GDX%datawritedone(wgdxHandle)
       rc = gdx.%GDX%datareaddone(gdxHandle)
   rc = gdx.%GDX%close(wgdxHandle)
   t = printT(flog, t, f, 'ctypes', 'reading and writing symbols str')
   assert rc == 0

   rc = gdx.%GDX%close(gdxHandle)
   assert rc == 0

gdx.xfree(byref(gdxHandle))
gdx.xfree(byref(wgdxHandle))

$offEmbeddedCode
$endIf

* C API Test
$ifThen %RUNCAPI% == 1
$  call cp "%gams.sysdir%apifiles/C/api/gdxcc.c" .
$  call cp "%gams.sysdir%apifiles/C/api/gdxcc.h" .
$  call cp "%gams.sysdir%apifiles/C/api/gclgms.h" .
$  ifThen.BC %system.BuildCode%==LEG
$     call which gcc > /dev/null
$     ifE.gcc errorLevel==0
$        call gcc -o libpfcgdx64.so -O -nostartfiles -shared -Wl,-Bsymbolic -pthread pfgdx.c gdxcc.c -I. -fPIC -fvisibility=hidden -DGC_NO_MUTEX
$     if not exist libpfcgdx64.so $log *** Could not build required C libary
$     if not exist libpfcgdx64.so $set RUNCAPI 0
$  elseIf.BC %system.BuildCode%==LAG
$     call which gcc > /dev/null
$     ifE.gcc errorLevel==0
$        call gcc -o libpfcgdx64.so -O -nostartfiles -shared -Wl,-Bsymbolic -pthread pfgdx.c gdxcc.c -I. -fPIC -fvisibility=hidden -DGC_NO_MUTEX
$     if not exist libpfcgdx64.so $log *** Could not build required C libary
$     if not exist libpfcgdx64.so $set RUNCAPI 0
$  elseIf.BC %system.BuildCode%==DAC
$     call which clang > /dev/null
$     ifE.clang errorLevel==0
$        call clang -o libpfcgdx64.dylib -O -dynamiclib -shared gdxcc.c pfgdx.c -ldl -fPIC -fvisibility=hidden -DGC_NO_MUTEX
$     if not exist libpfcgdx64.dylib $log *** Could not build required C libary
$     if not exist libpfcgdx64.dylib $set RUNCAPI 0
$  elseIf.BC %system.BuildCode%==DEG
$     call which gcc > /dev/null
$     ifE.gcc errorLevel==0
$        call gcc -o libpfcgdx64.dylib -O -dynamiclib -shared gdxcc.c pfgdx.c -ldl -fPIC -fvisibility=hidden -DGC_NO_MUTEX
$     if not exist libpfcgdx64.dylib $log *** Could not build required C libary
$     if not exist libpfcgdx64.dylib $set RUNCAPI 0
$  elseIf.BC %system.BuildCode%==WEI
$     call where -q icl > nul
$     ifE.icl errorLevel==0
$        call icl.exe -Fepfcgdx64.dll  -DGC_NO_MUTEX -nologo -MD -I. -O3 gdxcc.c pfgdx.c -LD -link -nodefaultlib:libc.lib
$     if not exist pfcgdx64.dll $log *** Could not build required C libary
$     if not exist pfcgdx64.dll $set RUNCAPI 0
$  else.BC
$     log *** Could not build required C libary (unknow buildCode %system.BuildCode%)
$     set RUNCAPI 0
$  endIf.BC
$endIf

$ifThen %RUNCAPI% == 1
$onEmbeddedCode Python:
from ctypes import *

system_directory = r'%gams.sysdir% '.rstrip()
if sys.platform == "linux" or sys.platform == "linux2":
    so_name = "libpfcgdx64.so"
elif sys.platform == "darwin":
    so_name = "libpfcgdx64.dylib"
elif sys.platform == "win32":
    so_name = "pfcgdx64.dll"
else:
    raise Exception(f'unknown OS {sys.platform}')
cgdx = cdll.LoadLibrary(os.path.join('./', so_name))
rc = cgdx.pfinit(c_char_p(system_directory.encode()))
assert rc == 0

for f in gams.get('fgdx'):
   t[1] = 0
   t[2] = time.time()
   rc = cgdx.pfopenread(c_char_p(f.encode()))
   t = printT(flog, t, f, 'capi', 'cgdxopenread')
   assert rc == 0

   rc = cgdx.pfreadraw(c_int(0))
   t = printT(flog, t, f, 'capi', 'reading symbols raw')
   assert rc == 0

   rc = cgdx.pfreadstr(c_int(0), c_int(0))
   t = printT(flog, t, f, 'capi', 'reading symbols str')
   assert rc == 0

   rc = cgdx.pfregister_uels(c_int(0),c_int(0))
   t = printT(flog, t, f, 'capi', 'register uels reading reverse=false')
   assert rc == 0

   rc = cgdx.pfreadmap()
   t = printT(flog, t, f, 'capi', 'reading symbols map reverse=false')
   assert rc == 0

   rc = cgdx.pfclose()
   assert rc == 0
   rc = cgdx.pfopenread(c_char_p(f.encode()))
   t = printT(flog, t, f, 'capi', 'cgdxopenread')
   assert rc == 0

   rc = cgdx.pfregister_uels(c_int(1),c_int(0))
   t = printT(flog, t, f, 'capi', 'register uels reading reverse=true')
   assert rc == 0

   rc = cgdx.pfreadmap()
   t = printT(flog, t, f, 'capi', 'reading symbols map reverse=true')
   assert rc == 0

   rc = cgdx.pfreadraw(c_int(1))
   t = printT(flog, t, f, 'capi', 'reading and writing symbols raw')
   assert rc == 0

   rc = cgdx.pfwritemap(c_int(0))
   t = printT(flog, t, f, 'capi', 'reading raw and writing map reverse=false')
   assert rc == 0

   rc = cgdx.pfwritemap(c_int(1))
   t = printT(flog, t, f, 'capi', 'reading raw and writing map reverse=true')
   assert rc == 0

   rc = cgdx.pfreadstr(c_int(0), c_int(1))
   t = printT(flog, t, f, 'capi', 'reading and writing symbols str reverse=false')
   assert rc == 0

   rc = cgdx.pfreadstr(c_int(1), c_int(1))
   t = printT(flog, t, f, 'capi', 'reading and writing symbols str reverse=true')
   assert rc == 0

   rc = cgdx.pfclose()
   assert rc == 0

rc = cgdx.pffini()
assert rc == 0

$offEmbeddedCode
$endIf

$onEmbeddedCode Python:
if '%LOGFILE%' != 'stdout' and '%LOGFILE%' != 'GAMSLOG':
   flog.close()
$offEmbeddedCode