$title 'Test Connect agent DomainWriter' (CADOMAINW,SEQ=911)

$onText
This test ensures the correctness of the Connect agent DomainWriter.

Contributor: Aileen Böhme, July 2022
$offText


$log --- Using Python library %sysEnv.GMSPYTHONLIB%

$onCheckErrorLevel
set i /i1*i5/, j /j1*j5/, k /k1*k5/;
set ii /i3*i4/;

$onEcho > t.gms
set i /i4*i8/, j /j1*j5,k1*k5/;
parameter a(i,j), b(i,j), c(*,*);
a(i,j)$(uniform(0,1)<0.5) = uniform(0,1);
b(i,j)$(uniform(0,1)<0.5) = uniform(0,1);
c(i,j)$(uniform(0,1)<0.5) = uniform(0,1);
$offEcho
$call.checkErrorLevel gams t.gms lo=%gams.lo% gdx x.gdx

$log Test error when number of domains specified not equal to true number of symbol domains
$onEmbeddedCode Connect:
- GAMSReader:
    symbols:
        - name: i
        - name: j
- GDXReader:
    file: x.gdx
    symbols:
        - name: a
- DomainWriter:
    symbols:
        - name: a(i,j,i)
$offEmbeddedCode
$if errorFree $abort 'Expect errors'
$clearErrors

$log Test error when there are spaces in domains specified
$onEmbeddedCode Connect:
- GAMSReader:
    symbols:
        - name: i
        - name: j
- GDXReader:
    file: x.gdx
    symbols:
        - name: a
- DomainWriter:
    symbols:
        - name: a(i," j")
$offEmbeddedCode
$if errorFree $abort 'Expect errors'
$clearErrors

$log test invalid option in symbols scope raising an exception
$onEmbeddedCode Connect:
- GDXReader:
    file: x.gdx
    symbols:
        - name: a
- DomainWriter:
    dropDomainViolations: True
    symbols:
        - name: a("i","j")
          invalidOption: invalid
$offEmbeddedCode
$if errorFree $abort 'Expect errors'
$clearErrors

$log create regular domain and drop domain violations
$onEmbeddedCode Connect:
- GAMSReader:
    symbols:
        - name: i
        - name: j
        - name: k
- GDXReader:
    file: x.gdx
    symbols:
        - name: a
        - name: b
- DomainWriter:
    dropDomainViolations: True
    symbols:
        - name: a(i,j)
        - name: b('i',k)
- PythonCode:
    code: |
        data = https://www.gams.com/latest/testlib_ml/connect.container["a"].records.values.tolist()
        expected = [['i4', 'j1', 0.843266708],
                    ['i4', 'j3', 0.292212117],
                    ['i4', 'j4', 0.34983050400000004],
                    ['i5', 'j1', 0.6397187590000001],
                    ['i5', 'j2', 0.250080533],
                    ['i5', 'j4', 0.35970026600000005],
                    ['i5', 'j5', 0.13149159000000002]]
        if data != expected:
            raise Exception("Unexpected data a.")
            
        data = https://www.gams.com/latest/testlib_ml/connect.container["b"].records.values.tolist()
        expected = [['i4', 'k1', 0.322086955],
                    ['i4', 'k4', 0.372888567],
                    ['i5', 'k2', 0.396121408],
                    ['i5', 'k3', 0.15237260800000002],
                    ['i5', 'k5', 0.13466312900000002],
                    ['i6', 'k1', 0.19213438200000002],
                    ['i6', 'k2', 0.5965581440000001],
                    ['i6', 'k4', 0.7831020040000001],
                    ['i7', 'k1', 0.6568921050000001],
                    ['i7', 'k3', 0.9867207240000001],
                    ['i7', 'k4', 0.6756549030000001],
                    ['i8', 'k1', 0.15034771600000002],
                    ['i8', 'k2', 0.18693790500000002],
                    ['i8', 'k5', 0.389378384]]
        if data != expected:
            raise Exception("Unexpected data b.")
$offEmbeddedCode

$log create regular domain and drop domain violations later
$onEmbeddedCode Connect:
- GAMSReader:
    symbols:
        - name: i
        - name: j
- GDXReader:
    file: x.gdx
    symbols:
        - name: a
        - name: b
- DomainWriter:
    symbols:
        - name: a(i,j)
        - name: b(i,'j')
- PythonCode:
    code: |
        import pandas as pd
        data = https://www.gams.com/latest/testlib_ml/pd.DataFrame(connect.container["a"].records.values.tolist())
        expected = pd.DataFrame([['i4', 'j1', 0.843266708],
                    ['i4', 'j3', 0.292212117],
                    ['i4', 'j4', 0.34983050400000004],
                    ['i4', 'k1', 0.500210669],
                    ['i5', 'j1', 0.6397187590000001],
                    ['i5', 'j2', 0.250080533],
                    ['i5', 'j4', 0.35970026600000005],
                    ['i5', 'j5', 0.13149159000000002],
                    ['i5', 'k1', 0.58911365],
                    ['i5', 'k3', 0.66573446],
                    ['i5', 'k5', 0.110492291],
                    ['i6', 'j2', 0.872462311],
                    ['i6', 'j3', 0.285814322],
                    ['i6', 'k2', 0.41330699400000004],
                    ['i6', 'k3', 0.31421226700000005],
                    ['i6', 'k4', 0.33855027200000004],
                    ['i6', 'k5', 0.645727127],
                    ['i7', 'j3', 0.661106261],
                    ['i7', 'k1', 0.086424624],
                    ['i7', 'k2', 0.6412511510000001],
                    ['i7', 'k4', 0.792360642],
                    ['i7', 'k5', 0.175661049],
                    ['i8', 'j3', 0.034140986000000005],
                    ['i8', 'k1', 0.35871415300000004],
                    ['i8', 'k2', 0.24642153900000002],
                    ['i8', 'k3', 0.93344972],
                    ['i8', 'k4', 0.783400461],
                    ['i8', 'k5', 0.125483222]])
        if not data.equals(expected):
            raise Exception("Unexpected data a.")
- DomainWriter:
    dropDomainViolations: True
    symbols: all
- PythonCode:
    code: |
        data = https://www.gams.com/latest/testlib_ml/connect.container["a"].records.values.tolist()
        expected = [['i4', 'j1', 0.843266708],
                    ['i4', 'j3', 0.292212117],
                    ['i4', 'j4', 0.34983050400000004],
                    ['i5', 'j1', 0.6397187590000001],
                    ['i5', 'j2', 0.250080533],
                    ['i5', 'j4', 0.35970026600000005],
                    ['i5', 'j5', 0.13149159000000002]]
        if data != expected:
            raise Exception("Unexpected data a.")
        
        data = https://www.gams.com/latest/testlib_ml/connect.container["b"].records.values.tolist()
        
        expected = [['i4', 'j2', 0.202015557],
                    ['i4', 'j3', 0.269613052],
                    ['i4', 'j4', 0.15128586900000002],
                    ['i4', 'j5', 0.330637734],
                    ['i4', 'k1', 0.322086955],
                    ['i4', 'k4', 0.372888567],
                    ['i5', 'j1', 0.913096325],
                    ['i5', 'j2', 0.735478889],
                    ['i5', 'j3', 0.576299805],
                    ['i5', 'j4', 0.006008368],
                    ['i5', 'j5', 0.5198811870000001],
                    ['i5', 'k2', 0.396121408],
                    ['i5', 'k3', 0.15237260800000002],
                    ['i5', 'k5', 0.13466312900000002]]
        if data != expected:
            raise Exception("Unexpected data b.")
$offEmbeddedCode

$log drop domain violations before
$onEmbeddedCode Connect:
- GAMSReader:
    symbols:
        - name: i
        - name: j
        - name: ii
- GDXReader:
    file: x.gdx
    symbols:
        - name: a
- DomainWriter:
    symbols:
        - name: a(i,j)
- DomainWriter:
    symbols:
        - name: a(ii,j)
          dropDomainViolations: before
- PythonCode:
    code: |
        import pandas as pd
        data = https://www.gams.com/latest/testlib_ml/pd.DataFrame(connect.container["a"].records.values.tolist())
        expected = pd.DataFrame([
            ['i4', 'j1', 0.843266708],
            ['i4', 'j3', 0.292212117],
            ['i4', 'j4', 0.34983050400000004],
            ['i5', 'j1', 0.6397187590000001],
            ['i5', 'j2', 0.250080533],
            ['i5', 'j4', 0.35970026600000005],
            ['i5', 'j5', 0.13149159000000002]
        ])
        if not data.equals(expected):
            raise Exception("Unexpected data a.")
$offEmbeddedCode

$log drop domain violations after
$onEmbeddedCode Connect:
- GAMSReader:
    symbols:
        - name: j
        - name: ii
- GDXReader:
    file: x.gdx
    symbols:
        - name: a
- DomainWriter:
    symbols:
        - name: a(ii,j)
          dropDomainViolations: after
- PythonCode:
    code: |
        import pandas as pd
        data = https://www.gams.com/latest/testlib_ml/pd.DataFrame(connect.container["a"].records.values.tolist())
        expected = pd.DataFrame([
            ['i4', 'j1', 0.843266708],
            ['i4', 'j3', 0.292212117],
            ['i4', 'j4', 0.34983050400000004]
        ])
        if not data.equals(expected):
            raise Exception("Unexpected data a.")
$offEmbeddedCode

$log check case (in)sensitivity of symbol and domain names I
$onEmbeddedCode Connect:
- GAMSReader:
    symbols:
        - name: i
        - name: j
        - name: k
- GDXReader:
    file: x.gdx
    symbols:
        - name: a
        - name: b
- DomainWriter:
    dropDomainViolations: True
    symbols:
        - name: A(i,j)
$offEmbeddedCode

$log check case (in)sensitivity of symbol and domain names II
$onEmbeddedCode Connect:
- GAMSReader:
    symbols:
        - name: i
        - name: j
        - name: k
- GDXReader:
    file: x.gdx
    symbols:
        - name: a
        - name: b
- DomainWriter:
    dropDomainViolations: True
    symbols:
        - name: a(I,j)
$offEmbeddedCode

$log check case (in)sensitivity of symbol and domain names III
$onEmbeddedCode Connect:
- GAMSReader:
    symbols:
        - name: i
        - name: j
        - name: k
- GDXReader:
    file: x.gdx
    symbols:
        - name: a
        - name: b
- DomainWriter:
    dropDomainViolations: True
    symbols:
        - name: A(I,j)
$offEmbeddedCode

$log check automatic adjustment of column names with relaxed domains
$onEmbeddedCode Connect:
- GDXReader:
    file: x.gdx
    symbols:
        - name: c
- DomainWriter:
    symbols:
        - name: c("i","j")
- PythonCode:
    code: |
      if connect.container['c'].domain != ["i", "j"]:
        raise Exception("Unexpected domain for c.")
      if connect.container['c'].records.columns.tolist() != ["i", "j", "value"]:
        raise Exception("Unexpected column names for c.")
$offEmbeddedCode

$log check automatic adjustment of column names with real domains
$onEmbeddedCode Connect:
- GDXReader:
    file: x.gdx
    symbols:
        - name: c
        - name: i
        - name: j
- DomainWriter:
    symbols:
        - name: c(i,j)
- PythonCode:
    code: |
      if connect.container['c'].domain != [connect.container['i'], connect.container['j']]:
        raise Exception("Unexpected domain for c.")
      if connect.container['c'].records.columns.tolist() != ["i", "j", "value"]:
        raise Exception("Unexpected column names for c.")
$offEmbeddedCode

$log check that None records symbol is changed to empty df with suitable domains (symbols: all)
$onEmbeddedCode Connect:
- PythonCode:
    code: |
      connect.container.addParameter("p", ["*", "*"])
- DomainWriter:
    symbols: all
- PythonCode:
    code: |
      if connect.container['p'].domain != ['*', '*']:
        raise Exception("Unexpected domain for p.")
      if connect.container['p'].records.columns.tolist() != ['uni_0', 'uni_1', 'value']:
        raise Exception("Unexpected column names for p.")
$offEmbeddedCode

$log check that None records symbol is changed to empty df with suitable domains (symbol)
$onEmbeddedCode Connect:
- GDXReader:
    file: x.gdx
    symbols:
        - name: i
        - name: j
- PythonCode:
    code: |
      connect.container.addParameter("p", ["*", "*"])
- DomainWriter:
    symbols:
      - name: p(i,j)
- PythonCode:
    code: |
      if connect.container['p'].domain != [connect.container['i'], connect.container['j']]:
        raise Exception("Unexpected domain for p.")
      if connect.container['p'].records.columns.tolist() != ["i", "j", "value"]:
        raise Exception("Unexpected column names for p.")
$offEmbeddedCode

$log check that user is allowed to add spaces between domains
$onEmbeddedCode Connect:
- GAMSReader:
    symbols:
        - name: i
        - name: j
- GDXReader:
    file: x.gdx
    symbols:
        - name: a
- DomainWriter:
    dropDomainViolations: True
    symbols:
        - name: a(i ,  "j2")
- PythonCode:
    code: |
      expected = [['i4', 'j1', 0.843266708],
                  ['i4', 'j3', 0.292212117],
                  ['i4', 'j4', 0.34983050400000004],
                  ['i4', 'k1', 0.500210669],
                  ['i5', 'j1', 0.6397187590000001],
                  ['i5', 'j2', 0.250080533],
                  ['i5', 'j4', 0.35970026600000005],
                  ['i5', 'j5', 0.13149159000000002],
                  ['i5', 'k1', 0.58911365],
                  ['i5', 'k3', 0.66573446],
                  ['i5', 'k5', 0.110492291]]
                
      if connect.container["a"].domain != [connect.container["i"], "j2"]:
        raise Exception("Unexpected domain for a.")
      if connect.container["a"].records.columns.tolist() != ["i", "j2", "value"]:
        raise Exception("Unexpected column names for a.")
      if connect.container["a"].records.values.tolist() != expected:
        raise Exception("Unexpected data a.")
$offEmbeddedCode


$log drop domain violations using before in root scope
$onEmbeddedCode Connect:
- GAMSReader:
    symbols:
        - name: i
        - name: j
        - name: ii
- GDXReader:
    file: x.gdx
    symbols:
        - name: a
- DomainWriter:
    symbols:
        - name: a(i,j)
- DomainWriter:
    dropDomainViolations: before
    symbols:
        - name: a(ii,j)
- PythonCode:
    code: |
        import pandas as pd
        data = https://www.gams.com/latest/testlib_ml/pd.DataFrame(connect.container["a"].records.values.tolist())
        expected = pd.DataFrame([
            ['i4', 'j1', 0.843266708],
            ['i4', 'j3', 0.292212117],
            ['i4', 'j4', 0.34983050400000004],
            ['i5', 'j1', 0.6397187590000001],
            ['i5', 'j2', 0.250080533],
            ['i5', 'j4', 0.35970026600000005],
            ['i5', 'j5', 0.13149159000000002]
        ])
        if not data.equals(expected):
            raise Exception("Unexpected data a.")
$offEmbeddedCode

$log drop domain violations using after in root scope
$onEmbeddedCode Connect:
- GAMSReader:
    symbols:
        - name: j
        - name: ii
- GDXReader:
    file: x.gdx
    symbols:
        - name: a
- DomainWriter:
    dropDomainViolations: after
    symbols:
        - name: a(ii,j)
- PythonCode:
    code: |
        import pandas as pd
        data = https://www.gams.com/latest/testlib_ml/pd.DataFrame(connect.container["a"].records.values.tolist())
        expected = pd.DataFrame([
            ['i4', 'j1', 0.843266708],
            ['i4', 'j3', 0.292212117],
            ['i4', 'j4', 0.34983050400000004]
        ])
        if not data.equals(expected):
            raise Exception("Unexpected data a.")
$offEmbeddedCode

$log drop domain violations using before (symbols scope) overwriting root scope
$onEmbeddedCode Connect:
- GAMSReader:
    symbols:
        - name: i
        - name: j
        - name: ii
- GDXReader:
    file: x.gdx
    symbols:
        - name: a
- DomainWriter:
    symbols:
        - name: a(i,j)
- DomainWriter:
    dropDomainViolations: True
    symbols:
        - name: a(ii,j)
          dropDomainViolations: before
- PythonCode:
    code: |
        import pandas as pd
        data = https://www.gams.com/latest/testlib_ml/pd.DataFrame(connect.container["a"].records.values.tolist())
        expected = pd.DataFrame([
            ['i4', 'j1', 0.843266708],
            ['i4', 'j3', 0.292212117],
            ['i4', 'j4', 0.34983050400000004],
            ['i5', 'j1', 0.6397187590000001],
            ['i5', 'j2', 0.250080533],
            ['i5', 'j4', 0.35970026600000005],
            ['i5', 'j5', 0.13149159000000002]
        ])
        if not data.equals(expected):
            raise Exception("Unexpected data a.")
$offEmbeddedCode