$title 'Test Connect agent Filter' (CAFILTER,SEQ=921)
$onText
This test ensures the correctness of the Connect agent Filter.

Contributor: Clemens Westphal, November 2022
$offText


$log --- Using Python library %sysEnv.GMSPYTHONLIB%


$onEcho > gen_data.gms
set l /l1_100, l1_200, l1_300, l2_100, l2_200, l2_300, l2_abc/;
alias(l,ll,lll);
set i /i1*i5, BAD_VALUE/;
set j /j1*j5, BAD_VALUE/;
set k /k1*k5, BAD_VALUE/;
set s2(i,j) /#i.#j/;

Scalar p0 / 1.5 /;

Parameter p1(i) /
'i1' 0.71747132,
'i2' 0.43266708,
'i3' 0.50375356,
'i4' 0.01137904,
'i5' EPS /;

Parameter p2(i,j) /
'i1'.'j1' EPS,
'i1'.'j2' +INF,
'i1'.'j3' -INF,
'i1'.'j4' 0.67113723,
'i1'.'j5' 0.00210669,
'i2'.'j1' 0.98117627,
'i2'.'j2' 0.78733378,
'i2'.'j3' 0.91133039,
'i2'.'j4' 0.62250467,
'i2'.'j5' 0.30692483,
'i3'.'j1' 0.39718759,
'i3'.'j2' 0.59517864,
'i3'.'j3' 0.50080533,
'i3'.'j4' 0.68928609,
'i3'.'j5' 0.35356381,
'i4'.'j1' 0.59700266,
'i4'.'j2' 0.51441368,
'i4'.'j3' 0.3149159,
'i4'.'j4' 0.50101788,
'i4'.'j5' 0.8911365,
'i5'.'j1' 0.30892812,
'i5'.'j2' 0.30815738,
'i5'.'j3' 0.6573446,
'i5'.'j4' 0.75857606,
'i5'.'j5' 0.03658477 /;

Parameter p3(i,j,k) /
'i1'.'j1'.'k2' EPS,
'i1'.'j1'.'k3' +INF,
'i2'.'j2'.'k3' -INF,
'i2'.'j2'.'k5' NA,
'i2'.'j4'.'k5' 0.93602205,
'i2'.'j5'.'k5' 0.55418475,
'i3'.'j1'.'k1' 0.5140711,
'i3'.'j3'.'k4' 0.88940325,
'i3'.'j5'.'k3' 0.11448928,
'i3'.'j5'.'k5' 0.45749415,
'i4'.'j1'.'k4' 0.59253884,
'i4'.'j3'.'k3' 0.34972611,
'i4'.'j5'.'k2' 0.75971873,
'i5'.'j1'.'k1' 0.26864289,
'i5'.'j1'.'k5' 0.10624414,
'i5'.'j3'.'k3' 0.01659033,
'i5'.'j4'.'k1' 0.35541754,
'i5'.'j5'.'k1' 0.40991774,
'i5'.'j5'.'k3' 0.32760523 /;

Parameter p3_2(l,ll,lll) /
'l1_100'.'l1_200'.'l1_200' 0.500210669,
'l1_100'.'l2_abc'.'l1_100' 0.338550272,
'l1_200'.'l1_100'.'l2_100' 0.102514669,
'l1_200'.'l1_100'.'l2_abc' 0.792360642,
'l1_200'.'l1_200'.'l1_100' 0.175661049,
'l1_200'.'l1_200'.'l2_200' 0.585131173,
'l1_200'.'l2_100'.'l2_100' 0.202015557,
'l1_200'.'l2_100'.'l2_200' 0.269613052,
'l1_200'.'l2_300'.'l2_abc' 0.576299805,
'l1_200'.'l2_abc'.'l1_100' 0.006008368,
'l1_300'.'l1_200'.'l2_100' 0.401346257,
'l1_300'.'l1_300'.'l2_200' 0.783102004,
'l1_300'.'l2_300'.'l2_abc' 0.150347716,
'l2_100'.'l1_100'.'l2_100' 0.187448731,
'l2_100'.'l1_100'.'l2_200' 0.540400638,
'l2_100'.'l1_200'.'l2_300' 0.543870155,
'l2_100'.'l1_300'.'l2_100' 0.177822574,
'l2_100'.'l1_300'.'l2_200' 0.016643898,
'l2_100'.'l2_100'.'l1_100' 0.196093864,
'l2_100'.'l2_300'.'l1_200' 0.948836169,
'l2_200'.'l1_100'.'l1_100' 0.368630572,
'l2_200'.'l1_100'.'l2_100' 0.84181978,
'l2_200'.'l1_200'.'l1_200' 0.773703434,
'l2_200'.'l2_100'.'l1_300' 0.485176103,
'l2_200'.'l2_100'.'l2_100' 0.698580858,
'l2_200'.'l2_200'.'l2_100' 0.263857554,
'l2_200'.'l2_300'.'l1_300' 0.32300194,
'l2_200'.'l2_abc'.'l2_300' 0.414599358,
'l2_300'.'l1_200'.'l1_300' 0.3741985,
'l2_300'.'l1_300'.'l1_200' 0.480878863,
'l2_300'.'l1_300'.'l2_200' 0.681039283,
'l2_300'.'l2_100'.'l2_abc' 0.57775716,
'l2_300'.'l2_300'.'l2_abc' 0.839795967,
'l2_abc'.'l1_100'.'l2_100' 0.010171892,
'l2_abc'.'l1_200'.'l1_300' 0.55303282,
'l2_abc'.'l1_300'.'l1_100' 0.430502537,
'l2_abc'.'l2_200'.'l1_200' 0.31014418,
'l2_abc'.'l2_200'.'l1_300' 0.82116568,
'l2_abc'.'l2_abc'.'l1_100' 0.686391574 /;

*UNDF
p3('i2', 'j4', 'k5') = 1/0;

free variable v0 /L 4.98 /;

positive variable v1(i) /
'i1'.L 0.6,
'i2'.M 0.28,
'i3'.L 0.4,
'i4'.M 2.62,
'i5'.M 2.42 /;

positive variable v2(i,j) /
'i1'.'BAD_VALUE'.M -0.217,
'i1'.'j1'.M -0.225,
'i1'.'j5'.M -0.195,
'i2'.'j1'.L 68.180,
'i2'.'j2'.M -0.225,
'i2'.'j3'.M -0.225,
'i3'.'j1'.M -0.225,
'i3'.'j2'.L 1.775,
'i3'.'j5'.L 826.796,
'i4'.'j1'.M -0.225,
'i4'.'j1'.Scale 2.0,
'i4'.'j5'.M -0.155,
'i5'.'j5'.L 609.007,
'BAD_VALUE'.'j2'.M -0.191 /;

equation e1(i) /
'i1'.L 0.072,
'i1'.LO 0,
'i1'.UP +Inf,
'i2'.L 0.1215,
'i2'.LO 0,
'i2'.UP +Inf,
'i3'.L 0.1781,
'i3'.LO 0,
'i3'.UP +Inf,
'i4'.L 0.2154,
'i4'.LO 0,
'i4'.UP +Inf,
'i5'.L 0.2524,
'i5'.LO 0,
'i5'.UP +Inf /;

equation e2(i,j) /
'BAD_VALUE'.'j2'.LO 0.0,
'i1'.'j1'.L 236.679,
'i1'.'j1'.LO -Inf,
'i1'.'j1'.UP 1866.25,
'i1'.'j4'.L 617.962,
'i1'.'j4'.LO -Inf,
'i1'.'j4'.UP 1866.25,
'i2'.'j2'.L 920.291,
'i2'.'j2'.LO -Inf,
'i2'.'j2'.UP 3011,
'i2'.'j3'.L 1360.278,
'i2'.'j3'.LO -Inf,
'i2'.'j3'.UP 3011,
'i4'.'j1'.L 193.046,
'i4'.'j1'.LO -Inf,
'i4'.'j1'.UP 11444.25,
'i4'.'j3'.L 63.317,
'i4'.'j3'.LO -Inf,
'i4'.'j3'.UP 11444.25,
'i4'.'j5'.L 2481.733,
'i4'.'j5'.LO -Inf,
'i4'.'j5'.UP 11444.25,
'i5'.'BAD_VALUE'.L 11.2,
'i5'.'j2'.UP 11141 /;

$offEcho

$call cd . && "%gams.sysdir%gams" gen_data.gms gdx=data > %system.nullFile%

set l, ll, lll;
set i, j, k;
set s2(i,j);
scalar p0;
parameter p1(i), p2(i,j), p3(i,j,k), p3_2(l,ll,lll);
positive variable v0, v1(i), v2(i,j);
equation e1(i), e2(i,j);

$onUNDF
$gdxIn data.gdx
$load l ll lll i j k s2 p0 p1 p2 p3 p3_2 v0 v1 v2 e1 e2
$gdxIn


***** Test label filters *****

$log test label filters using keep for 1-dim parameter
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'p1'}]
- Filter:
    name: p1
    newName: p1_new
    labelFilters:
      - dimension: 1
        keep: ['i2', 'i3']
- PythonCode:
    code: |
      expected = [
        ['i2', 0.43266708],
        ['i3', 0.50375356]
      ]
      if expected != connect.container['p1_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode

$log test label filters using reject for 1-dim parameter
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'p1'}]
- Filter:
    name: p1
    newName: p1_new
    labelFilters:
      - dimension: 1
        reject: ['i2', 'i3']
- PythonCode:
    code: |
      from gams import transfer as gt
      expected = [
        ['i1', 0.71747132],
        ['i4', 0.01137904],
        ['i5', gt.SpecialValues.EPS]
      ]
      if expected != connect.container['p1_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode

$log test label filters using keep for 2-dim parameter
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'p2'}]
- Filter:
    name: p2
    newName: p2_new
    labelFilters:
      - dimension: 1
        keep: ['i1', 'i2']
      - dimension: 2
        keep: ['j4', 'j5']
- PythonCode:
    code: |
      expected = [
        ['i1', 'j4', 0.67113723],
        ['i1', 'j5', 0.00210669],
        ['i2', 'j4', 0.62250467],
        ['i2', 'j5', 0.30692483]
      ]
      if expected != connect.container['p2_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode

$log test label filters using keep/reject for 2-dim parameter
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'p2'}]
- Filter:
    name: p2
    newName: p2_new
    labelFilters:
      - dimension: 1
        reject: ['i1', 'i2', 'i5']
      - dimension: 2
        keep: ['j1', 'j5']
- PythonCode:
    code: |
      expected = [
        ['i3', 'j1', 0.39718759],
        ['i3', 'j5', 0.35356381],
        ['i4', 'j1', 0.59700266],
        ['i4', 'j5', 0.8911365 ]
      ]
      if expected != connect.container['p2_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode

$log test label filters using keep for 3-dim parameter
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'p3'}]
- Filter:
    name: p3
    newName: p3_new
    labelFilters:
      - dimension: 1
        keep: ['i1', 'i2', 'i3']
      - dimension: 2
        keep: ['j2', 'j3']
      - dimension: 3
        keep: ['k4', 'k5']
- PythonCode:
    code: |
      from gams import transfer as gt
      expected = [
        ['i2', 'j2', 'k5', gt.SpecialValues.NA],
        ['i3', 'j3', 'k4', 0.88940325]
      ]
      
      # check NA manually since gt.SpecialValues.NA != gt.SpecialValues.NA
      data = connect.container['p3_new'].records.values.tolist()
      if expected[1:] != data[1:]:
        raise Exception("Unexpected Data")
      if expected[0][:-1] != data[0][:-1]:
        raise Exception("Unexpected Data")
      if not gt.SpecialValues.isNA(data[0][-1]): # check for NA
        raise Exception("Unexpected Data: NA")
$offEmbeddedCode

$log test label filters using keep/reject for 3-dim parameter
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'p3'}]
- Filter:
    name: p3
    newName: p3_new
    labelFilters:
      - dimension: 1
        keep: ['i1', 'i2', 'i3']
      - dimension: 2
        reject: ['j2', 'j3']
      - dimension: 3
        reject: ['k4', 'k5']
- PythonCode:
    code: |
      from gams import transfer as gt
      expected = [
        ['i1', 'j1', 'k2', gt.SpecialValues.EPS],
        ['i1', 'j1', 'k3', gt.SpecialValues.POSINF],
        ['i3', 'j1', 'k1', 0.5140711 ],
        ['i3', 'j5', 'k3', 0.11448928]
      ]
      if expected != connect.container['p3_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode

$log test label filters using keep for 1-dim set
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'i'}]
- Filter:
    name: i
    newName: i_new
    labelFilters:
      - dimension: 1
        keep: ['i1', 'i3', 'i5']
- PythonCode:
    code: |
      expected = [
        ['i1', ''],
        ['i3', ''],
        ['i5', '']
      ]
      if expected != connect.container['i_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode

$log test label filters using reject for 1-dim set
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'i'}]
- Filter:
    name: i
    newName: i_new
    labelFilters:
      - dimension: 1
        reject: ['i1', 'i3', 'i5', 'BAD_VALUE']
- PythonCode:
    code: |
      expected = [
        ['i2', ''],
        ['i4', '']
      ]
      if expected != connect.container['i_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode

$log test label filters using keep for 2-dim set
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 's2'}]
- Filter:
    name: s2
    newName: s2_new
    labelFilters:
      - dimension: 1
        reject: ['i1', 'i3', 'i5', 'BAD_VALUE']
      - dimension: 2
        keep: ['j1', 'j2']
- PythonCode:
    code: |
      expected = [
        ['i2', 'j1', ''],
        ['i2', 'j2', ''],
        ['i4', 'j1', ''],
        ['i4', 'j2', '']
      ]
      if expected != connect.container['s2_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode

$log test label filters using keep/reject for 2-dim set
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 's2'}]
- Filter:
    name: s2
    newName: s2_new
    labelFilters:
      - dimension: 1
        reject: ['i1', 'i3', 'i5', 'BAD_VALUE']
      - dimension: 2
        keep: ['j1', 'j2']
- PythonCode:
    code: |
      expected = [
        ['i2', 'j1', ''],
        ['i2', 'j2', ''],
        ['i4', 'j1', ''],
        ['i4', 'j2', '']
      ]
      if expected != connect.container['s2_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode

$log test label filters using keep for 1-dim variable
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'v1'}]
- Filter:
    name: v1
    newName: v1_new
    labelFilters:
      - dimension: 1
        keep: ['i3', 'i5']
- PythonCode:
    code: |
      from gams import transfer as gt
      expected = [
        ['i3', 0.4, 0.00, 0.0, gt.SpecialValues.POSINF, 1.0],
        ['i5', 0.0, 2.42, 0.0, gt.SpecialValues.POSINF, 1.0]
      ]
      if expected != connect.container['v1_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode

$log test label filters using reject for 1-dim variable
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'v1'}]
- Filter:
    name: v1
    newName: v1_new
    labelFilters:
      - dimension: 1
        reject: ['i1', 'i3', 'i5']
- PythonCode:
    code: |
      from gams import transfer as gt
      expected = [
        ['i2', 0.0, 0.28, 0.0, gt.SpecialValues.POSINF, 1.0],
        ['i4', 0.0, 2.62, 0.0, gt.SpecialValues.POSINF, 1.0]
      ]
      if expected != connect.container['v1_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode

$log test label filters using keep for 2-dim variable
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'v2'}]
- Filter:
    name: v2
    newName: v2_new
    labelFilters:
      - dimension: 1
        keep: ['i1', 'i3', 'i5']
      - dimension: 2
        keep: ['j2', 'j3', 'j4', 'BAD_VALUE']
- PythonCode:
    code: |
      from gams import transfer as gt
      expected = [
        ['i1', 'BAD_VALUE',   0.0, -0.217, 0.0, gt.SpecialValues.POSINF, 1.0],
        ['i3', 'j2'       , 1.775,    0.0, 0.0, gt.SpecialValues.POSINF, 1.0]
      ]
      if expected != connect.container['v2_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode


$log test label filters using keep/reject for 2-dim variable
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'v2'}]
- Filter:
    name: v2
    newName: v2_new
    labelFilters:
      - dimension: 1
        reject: ['i1', 'i3', 'i5', 'BAD_VALUE']
      - dimension: 2
        keep: ['j1', 'j5']
- PythonCode:
    code: |
      from gams import transfer as gt
      expected = [
        ['i2', 'j1', 68.180,  0.0  , 0.0, gt.SpecialValues.POSINF, 1.0],
        ['i4', 'j1',  0.0  , -0.225, 0.0, gt.SpecialValues.POSINF, 2.0],
        ['i4', 'j5',  0.0  , -0.155, 0.0, gt.SpecialValues.POSINF, 1.0]
      ]
      if expected != connect.container['v2_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode

$log test label filters using keep for 1-dim equation
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'e1'}]
- Filter:
    name: e1
    newName: e1_new
    labelFilters:
      - dimension: 1
        keep: ['i2', 'i3']
- PythonCode:
    code: |
      from gams import transfer as gt
      expected = [
        ['i2', 0.1215, 0.0, 0.0, gt.SpecialValues.POSINF, 1.0],
        ['i3', 0.1781, 0.0, 0.0, gt.SpecialValues.POSINF, 1.0]
      ]
      if expected != connect.container['e1_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode


$log test label filters using reject for 1-dim equation
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'e1'}]
- Filter:
    name: e1
    newName: e1_new
    labelFilters:
      - dimension: 1
        reject: ['i2', 'i3']
- PythonCode:
    code: |
      from gams import transfer as gt
      expected = [
        ['i1', 0.0720, 0.0, 0.0, gt.SpecialValues.POSINF, 1.0],
        ['i4', 0.2154, 0.0, 0.0, gt.SpecialValues.POSINF, 1.0],
        ['i5', 0.2524, 0.0, 0.0, gt.SpecialValues.POSINF, 1.0]
      ]
      if expected != connect.container['e1_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode

$log test label filters using keep for 2-dim equation
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'e2'}]
- Filter:
    name: e2
    newName: e2_new
    labelFilters:
      - dimension: 1
        keep: ['i1', 'i2']
      - dimension: 2
        keep: ['j3', 'j5']
- PythonCode:
    code: |
      from gams import transfer as gt
      expected = [
        ['i2', 'j3', 1360.278, 0.0, gt.SpecialValues.NEGINF, 3011.0, 1.0]
      ]
      if expected != connect.container['e2_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode

$log test label filters using keep/reject for 2-dim equation
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'e2'}]
- Filter:
    name: e2
    newName: e2_new
    labelFilters:
      - dimension: 1
        keep: ['i1', 'i2']
      - dimension: 2
        reject: ['j3', 'j2']
- PythonCode:
    code: |
      from gams import transfer as gt
      expected = [
        ['i1', 'j1', 236.679, 0.0, gt.SpecialValues.NEGINF, 1866.25, 1.0],
        ['i1', 'j4', 617.962, 0.0, gt.SpecialValues.NEGINF, 1866.25, 1.0]
      ]
      if expected != connect.container['e2_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode

$log test 'all' label filters for 3-dim parameter
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'p3_2'}]
- Filter:
    name: p3_2
    newName: p3_2_new
    labelFilters:
      - reject: ['l2_abc']
      - dimension: 1
        reject: ['l1_100']
      - dimension: 2
        regex: 'l1\w*'
      - dimension: 3
        keep: ['l2_100', 'l1_100', 'l2_200']
    valueFilters:
      - attribute: value
        rule: x<0.4
- PythonCode:
    code: |
      from gams import transfer as gt
      expected = [
        ['l1_200', 'l1_100', 'l2_100', 0.102514669],
        ['l1_200', 'l1_200', 'l1_100', 0.175661049],
        ['l2_100', 'l1_100', 'l2_100', 0.187448731],
        ['l2_100', 'l1_300', 'l2_100', 0.177822574],
        ['l2_100', 'l1_300', 'l2_200', 0.016643898],
        ['l2_200', 'l1_100', 'l1_100', 0.368630572]
      ]
      if expected != connect.container['p3_2_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode

$log test 'all' label filters for 2-dim set
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 's2'}]
- Filter:
    name: s2
    newName: s2_new
    labelFilters:
      - reject: ['BAD_VALUE']
- PythonCode:
    code: |
      from gams import transfer as gt
      expected = [
        ['i1', 'j1', ''],
        ['i1', 'j2', ''],
        ['i1', 'j3', ''],
        ['i1', 'j4', ''],
        ['i1', 'j5', ''],
        ['i2', 'j1', ''],
        ['i2', 'j2', ''],
        ['i2', 'j3', ''],
        ['i2', 'j4', ''],
        ['i2', 'j5', ''],
        ['i3', 'j1', ''],
        ['i3', 'j2', ''],
        ['i3', 'j3', ''],
        ['i3', 'j4', ''],
        ['i3', 'j5', ''],
        ['i4', 'j1', ''],
        ['i4', 'j2', ''],
        ['i4', 'j3', ''],
        ['i4', 'j4', ''],
        ['i4', 'j5', ''],
        ['i5', 'j1', ''],
        ['i5', 'j2', ''],
        ['i5', 'j3', ''],
        ['i5', 'j4', ''],
        ['i5', 'j5', '']
      ]
      if expected != connect.container['s2_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode

$log test 'all' label filters for 2-dim variable
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'v2'}]
- Filter:
    name: v2
    newName: v2_new
    labelFilters:
      - dimension: all
        reject: ['BAD_VALUE', 'i1', 'i2']
      - dimension: 2
        keep: ['j1']
- PythonCode:
    code: |
      from gams import transfer as gt
      expected = [
        ['i3', 'j1', 0.000, -0.225, 0.0, gt.SpecialValues.POSINF, 1.0],
        ['i4', 'j1', 0.000, -0.225, 0.0, gt.SpecialValues.POSINF, 2.0]
      ]
      if expected != connect.container['v2_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode


$log test 'all' label filters for 2-dim equation
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'e2'}]
- Filter:
    name: e2
    newName: e2_new
    labelFilters:
      - dimension: all
        reject: ['BAD_VALUE']
- PythonCode:
    code: |
      from gams import transfer as gt
      expected = [
        ['i1', 'j1',  236.679, 0.0, gt.SpecialValues.NEGINF,  1866.25, 1],
        ['i1', 'j4',  617.962, 0.0, gt.SpecialValues.NEGINF,  1866.25, 1],
        ['i2', 'j2',  920.291, 0.0, gt.SpecialValues.NEGINF,  3011.00, 1],
        ['i2', 'j3', 1360.278, 0.0, gt.SpecialValues.NEGINF,  3011.00, 1],
        ['i4', 'j1',  193.046, 0.0, gt.SpecialValues.NEGINF, 11444.25, 1],
        ['i4', 'j3',   63.317, 0.0, gt.SpecialValues.NEGINF, 11444.25, 1],
        ['i4', 'j5', 2481.733, 0.0, gt.SpecialValues.NEGINF, 11444.25, 1],
        ['i5', 'j2',    0.000, 0.0, gt.SpecialValues.NEGINF, 11141.00, 1]
      ]
      if expected != connect.container['e2_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode


***** Test value filters *****

$log test value filters for scalar
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'p0'}]
- Filter:
    name: p0
    newName: p0_new
    valueFilters:
      - attribute: value
        rule: x<1.5
- PythonCode:
    code: |
      expected = []
      if expected != connect.container['p0_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode

$log test value filters for 1-dim parameter
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'p1'}]
- Filter:
    name: p1
    newName: p1_new
    valueFilters:
      - attribute: value
        ruleIdentifier: val
        rule: val<0.5
- PythonCode:
    code: |
      from gams import transfer as gt
      expected = [
        ['i2', 0.43266708],
        ['i4', 0.01137904],
        ['i5', gt.SpecialValues.EPS]
      ]
      if expected != connect.container['p1_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode

$log test value filters for 2-dim parameter
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'p2'}]
- Filter:
    name: p2
    newName: p2_new
    valueFilters:
      - attribute: value
        rule: (x>0.2) & (x<0.5) | (x>=0.9)
- PythonCode:
    code: |
      from gams import transfer as gt
      expected = [
        ['i1', 'j1', gt.SpecialValues.EPS],
        ['i1', 'j2', gt.SpecialValues.POSINF],
        ['i1', 'j3', gt.SpecialValues.NEGINF],
        ['i2', 'j1', 0.98117627],
        ['i2', 'j3', 0.91133039],
        ['i2', 'j5', 0.30692483],
        ['i3', 'j1', 0.39718759],
        ['i3', 'j5', 0.35356381],
        ['i4', 'j3', 0.3149159 ],
        ['i5', 'j1', 0.30892812],
        ['i5', 'j2', 0.30815738]
      ]
      if expected != connect.container['p2_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode

$log test value filters for 3-dim parameter
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'p3'}]
- Filter:
    name: p3
    newName: p3_new
    valueFilters:
      - attribute: value
        rule: (x<0.2) | (x>0.8)
- PythonCode:
    code: |
      from gams import transfer as gt
      expected = [
        ['i1', 'j1', 'k2', gt.SpecialValues.EPS],
        ['i1', 'j1', 'k3', gt.SpecialValues.POSINF],
        ['i2', 'j2', 'k3', gt.SpecialValues.NEGINF],
        ['i2', 'j2', 'k5', gt.SpecialValues.NA],
        ['i2', 'j4', 'k5', gt.SpecialValues.UNDEF],
        ['i3', 'j3', 'k4', 0.88940325],
        ['i3', 'j5', 'k3', 0.11448928],
        ['i5', 'j1', 'k5', 0.10624414],
        ['i5', 'j3', 'k3', 0.01659033]
      ]
      # check UNDEF and NA manually since gt.SpecialValues.UNDEF/.NA != gt.SpecialValues.UNDEF/.NA
      data = connect.container['p3_new'].records.values.tolist()
      if expected[:3]+expected[5:] != data[:3]+data[5:]:
        raise Exception("Unexpected Data")
      if expected[3][:-1] != data[3][:-1]:
        raise Exception("Unexpected Data")
      if expected[4][:-1] != data[4][:-1]:
        raise Exception("Unexpected Data")
      if not gt.SpecialValues.isNA(data[3][-1]): # check for NA
        raise Exception("Unexpected Data: NA")
      if not gt.SpecialValues.isUndef(data[4][-1]): # check for UNDEF
        raise Exception("Unexpected Data: UNDEF")
$offEmbeddedCode

$log test value filters for scalar variable
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'v0'}]
- Filter:
    name: v0
    newName: v0_new
    valueFilters:
      - attribute: level
        rule: x>=5
- PythonCode:
    code: |
      expected = []
      if expected != connect.container['v0_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode

$log test value filters for 1-dim variable
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'v1'}]
- Filter:
    name: v1
    newName: v1_new
    valueFilters:
      - attribute: level
        rule: x==0
      - attribute: marginal
        ruleIdentifier: m
        rule: m>2
      - attribute: scale
        rule: x==1
- PythonCode:
    code: |
      from gams import transfer as gt
      expected = [
        ['i4', 0.0, 2.62, 0.0, gt.SpecialValues.POSINF, 1.0],
        ['i5', 0.0, 2.42, 0.0, gt.SpecialValues.POSINF, 1.0]
      ]
      if expected != connect.container['v1_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode

$log test value filters for 2-dim variable
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'v2'}]
- Filter:
    name: v2
    newName: v2_new
    valueFilters:
      - attribute: marginal
        rule: x<-0.2
      - attribute: level
        rule: (x>-0.01) & (x<0.01)
      - attribute: scale
        rule: x>1
- PythonCode:
    code: |
      from gams import transfer as gt
      expected = [
        ['i4', 'j1', 0.0, -0.225, 0.0, gt.SpecialValues.POSINF, 2.0]
      ]
      if expected != connect.container['v2_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode

$log test value filters for 1-dim equation
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'e1'}]
- Filter:
    name: e1
    newName: e1_new
    valueFilters:
      - attribute: level
        rule: (x>0.1) & (x<0.18) | (x>0.25)
- PythonCode:
    code: |
      from gams import transfer as gt
      expected = [
        ['i2', 0.1215, 0.0, 0.0, gt.SpecialValues.POSINF, 1.0],
        ['i3', 0.1781, 0.0, 0.0, gt.SpecialValues.POSINF, 1.0],
        ['i5', 0.2524, 0.0, 0.0, gt.SpecialValues.POSINF, 1.0]
      ]
      if expected != connect.container['e1_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode

$log test value filters for 2-dim equation
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'e2'}]
- Filter:
    name: e2
    newName: e2_new
    valueFilters:
      - attribute: level
        ruleIdentifier: v
        rule: v<200
      - attribute: marginal
        ruleIdentifier: m
        rule: m==0
      - attribute: upper
        rule: x>11444
- PythonCode:
    code: |
      from gams import transfer as gt
      expected = [
        ['i4',        'j1',       193.046, 0.0, gt.SpecialValues.NEGINF, 11444.25,                1.0],
        ['i4',        'j3',        63.317, 0.0, gt.SpecialValues.NEGINF, 11444.25,                1.0],
        ['i5',        'BAD_VALUE', 11.200, 0.0, gt.SpecialValues.NEGINF, gt.SpecialValues.POSINF, 1.0],
        ['BAD_VALUE', 'j2',         0.000, 0.0, 0.0,                     gt.SpecialValues.POSINF, 1.0]
      ]
      if expected != connect.container['e2_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode

$log test 'all' value filter for 3-dim parameter
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'p3'}]
- Filter:
    name: p3
    newName: p3_new
    valueFilters:
      - rule: (x<0.2) | (x>0.8)
- PythonCode:
    code: |
      from gams import transfer as gt
      expected = [
        ['i1', 'j1', 'k2', gt.SpecialValues.EPS],
        ['i1', 'j1', 'k3', gt.SpecialValues.POSINF],
        ['i2', 'j2', 'k3', gt.SpecialValues.NEGINF],
        ['i2', 'j2', 'k5', gt.SpecialValues.NA],
        ['i2', 'j4', 'k5', gt.SpecialValues.UNDEF],
        ['i3', 'j3', 'k4', 0.88940325],
        ['i3', 'j5', 'k3', 0.11448928],
        ['i5', 'j1', 'k5', 0.10624414],
        ['i5', 'j3', 'k3', 0.01659033]
      ]
      # check UNDEF and NA manually since gt.SpecialValues.UNDEF/.NA != gt.SpecialValues.UNDEF/.NA
      data = connect.container['p3_new'].records.values.tolist()
      if expected[:3]+expected[5:] != data[:3]+data[5:]:
        raise Exception("Unexpected Data")
      if expected[3][:-1] != data[3][:-1]:
        raise Exception("Unexpected Data")
      if expected[4][:-1] != data[4][:-1]:
        raise Exception("Unexpected Data")
      if not gt.SpecialValues.isNA(data[3][-1]): # check for NA
        raise Exception("Unexpected Data: NA")
      if not gt.SpecialValues.isUndef(data[4][-1]): # check for UNDEF
        raise Exception("Unexpected Data: UNDEF")
$offEmbeddedCode

$log test 'all' value filter for 2-dim variable
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'v2'}]
- Filter:
    name: v2
    newName: v2_new
    valueFilters:
      - rule: (x==0) | ((x>609) & (x<610)) | ((x>-0.2) & (x<0)) | (x==1)
- PythonCode:
    code: |
      from gams import transfer as gt
      expected = [
        ['i1', 'j5',   0.000, -0.195, 0.0, gt.SpecialValues.POSINF, 1.0],
        ['i4', 'j5',   0.000, -0.155, 0.0, gt.SpecialValues.POSINF, 1.0],
        ['i5', 'j5', 609.007,  0.000, 0.0, gt.SpecialValues.POSINF, 1.0],
        ['BAD_VALUE', 'j2',   0.000, -0.191, 0.0, gt.SpecialValues.POSINF, 1.0]
      ]
      if expected != connect.container['v2_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode

$log test 'all' value filter for 2-dim equation
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'e2'}]
- Filter:
    name: e2
    newName: e2_new
    labelFilters:
      - dimension: all
        reject: ['BAD_VALUE']
    valueFilters:
      - attribute: all
        rule: (x==0) | (x==1) | (x==11444.25) | ((x>1000) & (x<4000))
- PythonCode:
    code: |
      from gams import transfer as gt
      expected = [
        ['i2', 'j3', 1360.278, 0.0, gt.SpecialValues.NEGINF,  3011.00, 1.0],
        ['i4', 'j5', 2481.733, 0.0, gt.SpecialValues.NEGINF, 11444.25, 1.0]
      ]
      if expected != connect.container['e2_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode


***** Test label filters and value filters together *****

$log test value and label filters for 3-dim parameter
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'p3'}]
- Filter:
    name: p3
    newName: p3_new
    labelFilters:
      - dimension: 1
        keep: ['i1', 'i2', 'i3']
      - dimension: 3
        reject: ['k4', 'k5']
    valueFilters:
      - attribute: value
        rule: (x>0) & (x<0.5)
- PythonCode:
    code: |
      from gams import transfer as gt
      expected = [
        ['i1', 'j1', 'k2', gt.SpecialValues.EPS],
        ['i1', 'j1', 'k3', gt.SpecialValues.POSINF],
        ['i2', 'j2', 'k3', gt.SpecialValues.NEGINF],
        ['i3', 'j5', 'k3', 0.11448928]
      ]
      if expected != connect.container['p3_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode

$log test value and label filters for 2-dim variable
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'v2'}]
- Filter:
    name: v2
    newName: v2_new
    labelFilters:
      - dimension: 1
        reject: ['i1']
      - dimension: 2
        reject: ['j1', 'j2']
    valueFilters:
      - attribute: level
        rule: x<800
      - attribute: marginal
        rule: x<0
- PythonCode:
    code: |
      from gams import transfer as gt
      expected = [
        ['i2', 'j3', 0, -0.225, 0, gt.SpecialValues.POSINF, 1],
        ['i4', 'j5', 0, -0.155, 0, gt.SpecialValues.POSINF, 1]
      ]
      if expected != connect.container['v2_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode

$log test value and label filters for 3-dim equation
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'e2'}]
- Filter:
    name: e2
    newName: e2_new
    labelFilters:
      - dimension: 2
        keep: ['j1', 'j3', 'j4']
      - dimension: 1
        keep: ['i1', 'i4']
    valueFilters:
      - attribute: upper
        rule: x==1866.25
      - attribute: scale
        rule: x==1
- PythonCode:
    code: |
      from gams import transfer as gt
      expected = [
        ['i1', 'j1', 236.679, 0.0, gt.SpecialValues.NEGINF, 1866.25, 1.0],
        ['i1', 'j4', 617.962, 0.0, gt.SpecialValues.NEGINF, 1866.25, 1.0]
      ]
      if expected != connect.container['e2_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode


***** Test regular expressions *****

$log test regular expression of label filters for 3-dim parameter
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'p3'}]
- Filter:
    name: p3
    newName: p3_new
    labelFilters:
      - dimension: 1
        regex: 'i[1-3]'
      - dimension: 2
        reject: ['j2', 'j4']
      - dimension: 3
        regex: 'k[12]'
- PythonCode:
    code: |
      from gams import transfer as gt
      expected = [
        ['i1', 'j1', 'k2', gt.SpecialValues.EPS],
        ['i3', 'j1', 'k1', 0.5140711]
      ]
      if expected != connect.container['p3_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode

$log test regular expression of label filters for 3-dim parameter (2)
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'p3_2'}]
- Filter:
    name: p3_2
    newName: p3_2_new
    labelFilters:
      - dimension: 1
        reject: ['l1_300', 'l1_200', 'l1_100']
      - dimension: 2
        regex: '\w+abc'
      - dimension: 3
        regex: 'l2_\d*'
- PythonCode:
    code: |
      expected = [
        ['l2_200', 'l2_abc', 'l2_300', 0.414599358]
      ]
      if expected != connect.container['p3_2_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode

$log test regular expression of label filters for 3-dim parameter (3)
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'p3_2'}]
- Filter:
    name: p3_2
    newName: p3_2_new
    labelFilters:
      - dimension: 1
        regex: 'l\d_[1a]\w*'
      - dimension: 2
        keep: ['l2_abc', 'l1_300', 'l1_200', 'l1_100']
      - dimension: 3
        regex: 'l2\w[012]*'
    valueFilters:
      - attribute: value
        rule: x<0.1
- PythonCode:
    code: |
      expected = [
        ['l2_100', 'l1_300', 'l2_200', 0.016643898],
        ['l2_abc', 'l1_100', 'l2_100', 0.010171892]
      ]
      if expected != connect.container['p3_2_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode

$log test regular expression of label filters for 2-dim set
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 's2'}]
- Filter:
    name: s2
    newName: s2_new
    labelFilters:
      - dimension: 1
        regex: 'i[1-2]'
      - dimension: 2
        regex: '\w5'
- PythonCode:
    code: |
      expected = [
        ['i1', 'j5', ''],
        ['i2', 'j5', '']
      ]
      if expected != connect.container['s2_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode

$log test regular expression of label filters for 2-dim variable
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'v2'}]
- Filter:
    name: v2
    newName: v2_new
    labelFilters:
      - dimension: 1
        regex: '[i12]+'
      - dimension: 2
        regex: 'j[4-5]'
- PythonCode:
    code: |
      from gams import transfer as gt
      expected = [
        ['i1', 'j5', 0, -0.195, 0, gt.SpecialValues.POSINF, 1]
      ]
      if expected != connect.container['v2_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode

$log test regular expression of label filters for 2-dim equation
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'e2'}]
- Filter:
    name: e2
    newName: e2_new
    labelFilters:
      - dimension: 2
        regex: 'j1'
      - dimension: 1
        regex: 'i[2-5]'
- PythonCode:
    code: |
      from gams import transfer as gt
      expected = [
        ['i4', 'j1', 193.046, 0, gt.SpecialValues.NEGINF, 11444.25, 1]
      ]
      if expected != connect.container['e2_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode


***** Test special values *****

$log test filtering EPS
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'p3'}]
- Filter:
    name: p3
    newName: p3_new
    valueFilters:
      - attribute: value
        rejectSpecialValues: EPS
- PythonCode:
    code: |
      from gams import transfer as gt
      expected = [
        ['i1', 'j1', 'k3', gt.SpecialValues.POSINF],
        ['i2', 'j2', 'k3', gt.SpecialValues.NEGINF],
        ['i2', 'j2', 'k5', gt.SpecialValues.NA],
        ['i2', 'j4', 'k5', gt.SpecialValues.UNDEF],
        ['i2', 'j5', 'k5', 0.55418475],
        ['i3', 'j1', 'k1', 0.5140711],
        ['i3', 'j3', 'k4', 0.88940325],
        ['i3', 'j5', 'k3', 0.11448928],
        ['i3', 'j5', 'k5', 0.45749415],
        ['i4', 'j1', 'k4', 0.59253884],
        ['i4', 'j3', 'k3', 0.34972611],
        ['i4', 'j5', 'k2', 0.75971873],
        ['i5', 'j1', 'k1', 0.26864289],
        ['i5', 'j1', 'k5', 0.10624414],
        ['i5', 'j3', 'k3', 0.01659033],
        ['i5', 'j4', 'k1', 0.35541754],
        ['i5', 'j5', 'k1', 0.40991774],
        ['i5', 'j5', 'k3', 0.32760523]
      ]
      # check NA/UNDF manually since gt.SpecialValues.NA != gt.SpecialValues.NA
      data = https://www.gams.com/latest/testlib_ml/connect.container['p3_new'].records.values.tolist()
      if expected[:2] != data[:2]:
        raise Exception("Unexpected Data")
      if expected[4:] != data[4:]:
        raise Exception("Unexpected Data")
      if expected[2][:-1] != data[2][:-1]:
        raise Exception("Unexpected Data")
      if expected[3][:-1] != data[3][:-1]:
        raise Exception("Unexpected Data")
      if not gt.SpecialValues.isNA(data[2][-1]): # check for NA
        raise Exception("Unexpected Data: NA")
      if not gt.SpecialValues.isUndef(data[3][-1]): # check for UNDF
        raise Exception("Unexpected Data: UNDF")
$offEmbeddedCode

$log test filtering INF
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'p3'}]
- Filter:
    name: p3
    newName: p3_new
    valueFilters:
      - attribute: value
        rejectSpecialValues: INF
- PythonCode:
    code: |
      from gams import transfer as gt
      expected = [
        ['i1', 'j1', 'k2', gt.SpecialValues.EPS],
        ['i2', 'j2', 'k3', gt.SpecialValues.NEGINF],
        ['i2', 'j2', 'k5', gt.SpecialValues.NA],
        ['i2', 'j4', 'k5', gt.SpecialValues.UNDEF],
        ['i2', 'j5', 'k5', 0.55418475],
        ['i3', 'j1', 'k1', 0.5140711],
        ['i3', 'j3', 'k4', 0.88940325],
        ['i3', 'j5', 'k3', 0.11448928],
        ['i3', 'j5', 'k5', 0.45749415],
        ['i4', 'j1', 'k4', 0.59253884],
        ['i4', 'j3', 'k3', 0.34972611],
        ['i4', 'j5', 'k2', 0.75971873],
        ['i5', 'j1', 'k1', 0.26864289],
        ['i5', 'j1', 'k5', 0.10624414],
        ['i5', 'j3', 'k3', 0.01659033],
        ['i5', 'j4', 'k1', 0.35541754],
        ['i5', 'j5', 'k1', 0.40991774],
        ['i5', 'j5', 'k3', 0.32760523]
      ]
      # check NA/UNDF manually since gt.SpecialValues.NA != gt.SpecialValues.NA
      data = https://www.gams.com/latest/testlib_ml/connect.container['p3_new'].records.values.tolist()
      if expected[:2] != data[:2]:
        raise Exception("Unexpected Data")
      if expected[4:] != data[4:]:
        raise Exception("Unexpected Data")
      if expected[2][:-1] != data[2][:-1]:
        raise Exception("Unexpected Data")
      if expected[3][:-1] != data[3][:-1]:
        raise Exception("Unexpected Data")
      if not gt.SpecialValues.isNA(data[2][-1]): # check for NA
        raise Exception("Unexpected Data: NA")
      if not gt.SpecialValues.isUndef(data[3][-1]): # check for UNDF
        raise Exception("Unexpected Data: UNDF")
$offEmbeddedCode

$log test filtering -INF
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'p3'}]
- Filter:
    name: p3
    newName: p3_new
    valueFilters:
      - attribute: value
        rejectSpecialValues: -INF
- PythonCode:
    code: |
      from gams import transfer as gt
      expected = [
        ['i1', 'j1', 'k2', gt.SpecialValues.EPS],
        ['i1', 'j1', 'k3', gt.SpecialValues.POSINF],
        ['i2', 'j2', 'k5', gt.SpecialValues.NA],
        ['i2', 'j4', 'k5', gt.SpecialValues.UNDEF],
        ['i2', 'j5', 'k5', 0.55418475],
        ['i3', 'j1', 'k1', 0.5140711],
        ['i3', 'j3', 'k4', 0.88940325],
        ['i3', 'j5', 'k3', 0.11448928],
        ['i3', 'j5', 'k5', 0.45749415],
        ['i4', 'j1', 'k4', 0.59253884],
        ['i4', 'j3', 'k3', 0.34972611],
        ['i4', 'j5', 'k2', 0.75971873],
        ['i5', 'j1', 'k1', 0.26864289],
        ['i5', 'j1', 'k5', 0.10624414],
        ['i5', 'j3', 'k3', 0.01659033],
        ['i5', 'j4', 'k1', 0.35541754],
        ['i5', 'j5', 'k1', 0.40991774],
        ['i5', 'j5', 'k3', 0.32760523]
      ]
      # check NA/UNDF manually since gt.SpecialValues.NA != gt.SpecialValues.NA
      data = https://www.gams.com/latest/testlib_ml/connect.container['p3_new'].records.values.tolist()
      if expected[:2] != data[:2]:
        raise Exception("Unexpected Data")
      if expected[4:] != data[4:]:
        raise Exception("Unexpected Data")
      if expected[2][:-1] != data[2][:-1]:
        raise Exception("Unexpected Data")
      if expected[3][:-1] != data[3][:-1]:
        raise Exception("Unexpected Data")
      if not gt.SpecialValues.isNA(data[2][-1]): # check for NA
        raise Exception("Unexpected Data: NA")
      if not gt.SpecialValues.isUndef(data[3][-1]): # check for UNDF
        raise Exception("Unexpected Data: UNDF")
$offEmbeddedCode

$log test filtering NA
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'p3'}]
- Filter:
    name: p3
    newName: p3_new
    valueFilters:
      - attribute: value
        rejectSpecialValues: NA
- PythonCode:
    code: |
      from gams import transfer as gt
      expected = [
        ['i1', 'j1', 'k2', gt.SpecialValues.EPS],
        ['i1', 'j1', 'k3', gt.SpecialValues.POSINF],
        ['i2', 'j2', 'k3', gt.SpecialValues.NEGINF],
        ['i2', 'j4', 'k5', gt.SpecialValues.UNDEF],
        ['i2', 'j5', 'k5', 0.55418475],
        ['i3', 'j1', 'k1', 0.5140711],
        ['i3', 'j3', 'k4', 0.88940325],
        ['i3', 'j5', 'k3', 0.11448928],
        ['i3', 'j5', 'k5', 0.45749415],
        ['i4', 'j1', 'k4', 0.59253884],
        ['i4', 'j3', 'k3', 0.34972611],
        ['i4', 'j5', 'k2', 0.75971873],
        ['i5', 'j1', 'k1', 0.26864289],
        ['i5', 'j1', 'k5', 0.10624414],
        ['i5', 'j3', 'k3', 0.01659033],
        ['i5', 'j4', 'k1', 0.35541754],
        ['i5', 'j5', 'k1', 0.40991774],
        ['i5', 'j5', 'k3', 0.32760523]
      ]
      # check UNDF manually since gt.SpecialValues.NA != gt.SpecialValues.NA
      data = https://www.gams.com/latest/testlib_ml/connect.container['p3_new'].records.values.tolist()
      if expected[:3] != data[:3]:
        raise Exception("Unexpected Data")
      if expected[4:] != data[4:]:
        raise Exception("Unexpected Data")
      if expected[3][:-1] != data[3][:-1]:
        raise Exception("Unexpected Data")
      if not gt.SpecialValues.isUndef(data[3][-1]): # check for UNDF
        raise Exception("Unexpected Data: UNDF")
$offEmbeddedCode

$log test filtering UNDF
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'p3'}]
- Filter:
    name: p3
    newName: p3_new
    valueFilters:
      - attribute: value
        rejectSpecialValues: UNDEF
- PythonCode:
    code: |
      from gams import transfer as gt
      expected = [
        ['i1', 'j1', 'k2', gt.SpecialValues.EPS],
        ['i1', 'j1', 'k3', gt.SpecialValues.POSINF],
        ['i2', 'j2', 'k3', gt.SpecialValues.NEGINF],
        ['i2', 'j2', 'k5', gt.SpecialValues.NA],
        ['i2', 'j5', 'k5', 0.55418475],
        ['i3', 'j1', 'k1', 0.5140711],
        ['i3', 'j3', 'k4', 0.88940325],
        ['i3', 'j5', 'k3', 0.11448928],
        ['i3', 'j5', 'k5', 0.45749415],
        ['i4', 'j1', 'k4', 0.59253884],
        ['i4', 'j3', 'k3', 0.34972611],
        ['i4', 'j5', 'k2', 0.75971873],
        ['i5', 'j1', 'k1', 0.26864289],
        ['i5', 'j1', 'k5', 0.10624414],
        ['i5', 'j3', 'k3', 0.01659033],
        ['i5', 'j4', 'k1', 0.35541754],
        ['i5', 'j5', 'k1', 0.40991774],
        ['i5', 'j5', 'k3', 0.32760523]
      ]
      # check NA manually since gt.SpecialValues.NA != gt.SpecialValues.NA
      data = https://www.gams.com/latest/testlib_ml/connect.container['p3_new'].records.values.tolist()
      if expected[:3] != data[:3]:
        raise Exception("Unexpected Data")
      if expected[4:] != data[4:]:
        raise Exception("Unexpected Data")
      if expected[3][:-1] != data[3][:-1]:
        raise Exception("Unexpected Data")
      if not gt.SpecialValues.isNA(data[3][-1]): # check for NA
        raise Exception("Unexpected Data: NA")
$offEmbeddedCode

$log test filtering EPS, NEGINF, UNDF
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'p3'}]
- Filter:
    name: p3
    newName: p3_new
    valueFilters:
      - attribute: value
        rejectSpecialValues: [EPS, -INF, UNDEF]
- PythonCode:
    code: |
      from gams import transfer as gt
      expected = [
        ['i1', 'j1', 'k3', gt.SpecialValues.POSINF],
        ['i2', 'j2', 'k5', gt.SpecialValues.NA],
        ['i2', 'j5', 'k5', 0.55418475],
        ['i3', 'j1', 'k1', 0.5140711],
        ['i3', 'j3', 'k4', 0.88940325],
        ['i3', 'j5', 'k3', 0.11448928],
        ['i3', 'j5', 'k5', 0.45749415],
        ['i4', 'j1', 'k4', 0.59253884],
        ['i4', 'j3', 'k3', 0.34972611],
        ['i4', 'j5', 'k2', 0.75971873],
        ['i5', 'j1', 'k1', 0.26864289],
        ['i5', 'j1', 'k5', 0.10624414],
        ['i5', 'j3', 'k3', 0.01659033],
        ['i5', 'j4', 'k1', 0.35541754],
        ['i5', 'j5', 'k1', 0.40991774],
        ['i5', 'j5', 'k3', 0.32760523]
      ]
      # check NA manually since gt.SpecialValues.NA != gt.SpecialValues.NA
      data = https://www.gams.com/latest/testlib_ml/connect.container['p3_new'].records.values.tolist()
      if expected[:1] != data[:1]:
        raise Exception("Unexpected Data")
      if expected[2:] != data[2:]:
        raise Exception("Unexpected Data")
      if expected[1][:-1] != data[1][:-1]:
        raise Exception("Unexpected Data")
      if not gt.SpecialValues.isNA(data[1][-1]): # check for NA
        raise Exception("Unexpected Data: NA")
$offEmbeddedCode

$log test filtering EPS, NEGINF, UNDF with additional rule
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'p3'}]
- Filter:
    name: p3
    newName: p3_new
    valueFilters:
      - attribute: value
        rule: (x<0.2) | (x>0.8)
        rejectSpecialValues: [EPS, -INF, UNDEF]
- PythonCode:
    code: |
      from gams import transfer as gt
      expected = [
        ['i1', 'j1', 'k3', gt.SpecialValues.POSINF],
        ['i2', 'j2', 'k5', gt.SpecialValues.NA],
        ['i3', 'j3', 'k4', 0.88940325],
        ['i3', 'j5', 'k3', 0.11448928],
        ['i5', 'j1', 'k5', 0.10624414],
        ['i5', 'j3', 'k3', 0.01659033],
      ]
      # check NA manually since gt.SpecialValues.NA != gt.SpecialValues.NA
      data = connect.container['p3_new'].records.values.tolist()
      if expected[:1] != data[:1]:
        raise Exception("Unexpected Data")
      if expected[2:] != data[2:]:
        raise Exception("Unexpected Data")
      if expected[1][:-1] != data[1][:-1]:
        raise Exception("Unexpected Data")
      if not gt.SpecialValues.isNA(data[1][-1]): # check for NA
        raise Exception("Unexpected Data: NA")
$offEmbeddedCode

$log test filtering with attribute: all does not fail if the underlying DataFrame becomes empty before reaching the last attribute (scale)
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'v1'}]
- Filter:
    name: v1
    newName: v1_new
    valueFilters:
      - attribute: all
        rejectSpecialValues: INF
- PythonCode:
    code: |
      data = connect.container['v1_new'].records.values.tolist()
      if data != []:
        raise Exception("Unexpected Data")
$offEmbeddedCode

$if not errorFree $abort 'Errors'


***** Test invalid input *****

$log test duplicate dimensions (1)
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'p2'}]
- Filter:
    name: p2
    newName: p2_new
    labelFilters:
      - dimension: 2
        keep: ['j1', 'j2']
      - dimension: 2
        keep: ['j3', 'j4']
$offEmbeddedCode
$if errorFree $abort 'Expect errors for duplicate columns (1)'
$clearErrors

$log test duplicate attributes (2)
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'p2'}]
- Filter:
    name: p2
    newName: p2_new
    valueFilters:
      - attribute: value
        rule: x>0
      - attribute: value
        rule: x>0
$offEmbeddedCode
$if errorFree $abort 'Expect errors for duplicate attributes (2)'
$clearErrors

$log test wrong dimension
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'p2'}]
- Filter:
    name: p2
    newName: p2_new
    labelFilters:
      - dimension: 3
        keep: ['j1']
$offEmbeddedCode
$if errorFree $abort 'Expect errors for wrong dimension'
$clearErrors

$log test wrong dimension > 20
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'p2'}]
- Filter:
    name: p2
    newName: p2_new
    labelFilters:
      - dimension: 21
        keep: ['j1']
$offEmbeddedCode
$if errorFree $abort 'Expect errors for wrong dimension > 20'
$clearErrors

$log test wrong attribute name
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'p2'}]
- Filter:
    name: p2
    newName: p2_new
    valueFilters:
      - attribute: doesnt_exist
        rule: x>0
$offEmbeddedCode
$if errorFree $abort 'Expect errors for wrong attribute name'
$clearErrors

$log test writing to same symbol
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'p2'}]
- Filter:
    name: p2
    newName: p2
    labelFilters:
      - dimension: 2
        keep: ['j1', 'j2']
$offEmbeddedCode
$if errorFree $abort 'Expect errors when writing to same symbol'
$clearErrors

$log test value filter for sets
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 's2'}]
- Filter:
    name: s2
    newName: s2_new
    valueFilters:
      - attribute: value
        rule: x>0
$offEmbeddedCode
$if errorFree $abort 'Expect errors when using value filter for sets'
$clearErrors

$log test level as value filter for parameter
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'p2'}]
- Filter:
    name: p2
    newName: p2_new
    valueFilters:
      - attribute: level
        rule: x>0
$offEmbeddedCode
$if errorFree $abort 'Expect errors when using level as value filter for parameter'
$clearErrors

$log test value as value filter for variable
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'v1'}]
- Filter:
    name: v1
    newName: v1_new
    valueFilters:
      - attribute: value
        rule: x>0
$offEmbeddedCode
$if errorFree $abort 'Expect errors when using value as value filter for variable'
$clearErrors

$log test value filters for set
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 's2'}]
- Filter:
    name: s2
    newName: s2_new
    valueFilters:
      - attribute: all
        rule: x>0
$offEmbeddedCode
$if errorFree $abort 'Expect errors when using value filters for set'
$clearErrors

$log test using existing name
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'p1'}, {'name': 'p2'}]
- Filter:
    name: p1
    newName: P2
    labelFilters:
      - dimension: 1
        keep: ['i2', 'i3']
$offEmbeddedCode
$if errorFree $abort 'Expect errors when using an existing name'
$clearErrors

$log test invalid option in labelFilters scope raising an exception
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'p1'}]
- Filter:
    name: p1
    newName: p1_new
    labelFilters:
      - dimension: 1
        keep: ['i2', 'i3']
        invalidOption: invalid
$offEmbeddedCode
$if errorFree $abort 'Expect errors for invalid option in labelFilters scope'
$clearErrors

$log test invalid option in valueFilters scope raising an exception
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'p1'}]
- Filter:
    name: p1
    newName: p1_new
    valueFilters:
      - attribute: all
        rule: x>0
        invalidOption: invalid
$offEmbeddedCode
$if errorFree $abort 'Expect errors for invalid option in valueFilters scope'
$clearErrors

***** Other tests *****

$log test empty label filter using keep
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'p2'}]
- Filter:
    name: p2
    newName: p2_new
    labelFilters:
      - dimension: 1
        keep: []
      - dimension: 2
        keep: []
- PythonCode:
    code: |
      expected = [
      ]
      if expected != connect.container['p2_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode

$log test empty label filter using reject
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'p1'}]
- Filter:
    name: p1
    newName: p1_new
    labelFilters:
      - dimension: 1
        reject: []
- PythonCode:
    code: |
      from gams import transfer as gt
      expected = [
        ['i1', 0.71747132],
        ['i2', 0.43266708],
        ['i3', 0.50375356],
        ['i4', 0.01137904],
        ['i5', gt.SpecialValues.EPS]
      ]
      if expected != connect.container['p1_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode

$log test empty value filter
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'p1'}]
- Filter:
    name: p1
    newName: p1_new
    valueFilters:
      - attribute: value
        rule: ''
- PythonCode:
    code: |
      from gams import transfer as gt
      expected = [
        ['i1', 0.71747132],
        ['i2', 0.43266708],
        ['i3', 0.50375356],
        ['i4', 0.01137904],
        ['i5', gt.SpecialValues.EPS]
      ]
      if expected != connect.container['p1_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode

$log test no filters
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'p1'}]
- Filter:
    name: p1
    newName: p1_new
- PythonCode:
    code: |
      from gams import transfer as gt
      expected = [
        ['i1', 0.71747132],
        ['i2', 0.43266708],
        ['i3', 0.50375356],
        ['i4', 0.01137904],
        ['i5', gt.SpecialValues.EPS]
      ]
      if expected != connect.container['p1_new'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode

$log test that original symbol has no changes
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'p1'}]
- Filter:
    name: p1
    newName: p1_new
    labelFilters:
      - dimension: 1
        keep: ['i1']
- PythonCode:
    code: |
      from gams import transfer as gt
      expected = [
        ['i1', 0.71747132],
        ['i2', 0.43266708],
        ['i3', 0.50375356],
        ['i4', 0.01137904],
        ['i5', gt.SpecialValues.EPS]
      ]
      if expected != connect.container['p1'].records.values.tolist():
        raise Exception("Unexpected Data")
$offEmbeddedCode

$log test when records is empty, creates a new symbol with records being empty
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: [{'name': 'p1'}]
- Filter:
    name: p1
    newName: p1_new
    labelFilters:
      - dimension: 1
        keep: ['x']
- Filter:
    name: p1_new
    newName: p1_2_new
    labelFilters:
      - dimension: 1
        keep: ['i1']
- PythonCode:
    code: |
      if not connect.container['p1_2_new'].records.empty:
        raise Exception("Expected empty Data")
$offEmbeddedCode

$log test when records=None, creates a new symbol with empty DataFrame
$onEmbeddedCode Connect:
- PythonCode:
    code: |
      connect.container.addSet("p4", ["*", "*", "i"])
- Filter:
    name: p4
    newName: p4_new
    labelFilters:
      - dimension: 1
        keep: ['i1']
- PythonCode:
    code: |
      data = connect.container['p4_new'].records
      expected_cols = ['uni_0', 'uni_1', 'i_2', 'element_text']
          
      if data is None or not data.empty:
        raise Exception("Expected >p4_new< to have an empty DataFrame.")

      if (data.columns != expected_cols).any():
        raise Exception("Unexpected columns for >p4_new<.")
$offEmbeddedCode