Four turbine flexible setup

.gitignore

@@ -0,0 +1 @@
+*.exo.*

exawind/NREL_5MW_Turbine/flexible/input_files/5MW_Land_BD_DLL_WTurb/NRELOffshrBsline5MW_Onshore_ServoDyn.dat

@@ -74,7 +74,7 @@ True          GenTiStp     - Method to stop the generator {T: timed using TimGen
 ---------------------- CABLE CONTROL -------------------------------------------
           0   CCmode       - Cable control mode {0: none, 4: user-defined from Simulink/Labview, 5: user-defined from Bladed-style DLL} (switch)
 ---------------------- BLADED INTERFACE ---------------------------------------- [used only with Bladed Interface]
-"/projects/wind/hpc/wind/ndeveld/em-exawind-ghost/spack/opt/spack/linux-rhel8-broadwell/intel-2023.2.0/rosco-2.9.0-mg6c46gm4mmctfasbydcenh43f6hxi3x/lib/libdiscon.so"    DLL_FileName - Name/location of the dynamic library {.dll [Windows] or .so [Linux]} in the Bladed-DLL format (-) [used only with Bladed Interface]
+"/lustre/orion/cfd162/proj-shared/marchdf/exawind/exawind-manager/spack/opt/spack/linux-sles15-zen3/clang-17.0.0/rosco-2.9.0-gneu3wcmld5ck4ferzh425xkwylsvwfz/lib/libdiscon.so"    DLL_FileName - Name/location of the dynamic library {.dll [Windows] or .so [Linux]} in the Bladed-DLL format (-) [used only with Bladed Interface]
 "DISCON.IN"    DLL_InFile   - Name of input file sent to the DLL (-) [used only with Bladed Interface]
 "DISCON"      DLL_ProcName - Name of procedure in DLL to be called (-) [case sensitive; used only with DLL Interface]
 "default"     DLL_DT       - Communication interval for dynamic library (s) (or "default") [used only with Bladed Interface]

exawind/NREL_5MW_Turbine/flexible/input_files/frontier_all.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+#SBATCH -J hybrid
+#SBATCH -o %x.o%j
+#SBATCH --account=CFD162
+#SBATCH --time=48:00:00
+#SBATCH --nodes=32
+#SBATCH -S 0
+# #SBATCH --partition=extended
+
+#SBATCH --time=00:30:00
+
+set -e
+cmd() {
+  echo "+ $@"
+  eval "$@"
+}
+
+cmd "module unload PrgEnv-cray"
+cmd "module load PrgEnv-gnu-amd/8.6.0"
+cmd "module load amd-mixed/6.1.3"
+cmd "module load cray-python"
+cmd "export FI_MR_CACHE_MONITOR=memhooks"
+cmd "export FI_CXI_RX_MATCH_MODE=software"
+cmd "export MPICH_SMP_SINGLE_COPY_MODE=NONE"
+cmd "export MPICH_GPU_SUPPORT_ENABLED=0"
+cmd "export EXAWIND_MANAGER=${HOME}/exawind/exawind-manager"
+cmd "source ${EXAWIND_MANAGER}/start.sh && spack-start"
+cmd "spack env activate -d ${EXAWIND_MANAGER}/environments/exawind-dev"
+cmd "spack load exawind+amr_wind_gpu~nalu_wind_gpu"
+
+AWIND_RANK_PER_NODE=8
+NWIND_RANK_PER_NODE=56
+NWIND_SOLVERS=1
+
+cmd "srun -n 1 openfastcpp inp.yaml > precursor.log 2>&1"
+
+STK_BALANCE_RANKS=$((${SLURM_JOB_NUM_NODES}*${NWIND_RANK_PER_NODE}/${NWIND_SOLVERS}))
+MESH_FILE="../mesh/split_tower_and_blades.exo"
+cmd "rm -f ${MESH_FILE}.*"
+cmd "rm -f split_tower_and_blades.1_to_${STK_BALANCE_RANKS}.log"
+cmd "srun -N${SLURM_JOB_NUM_NODES} -n${STK_BALANCE_RANKS} stk_balance.exe ${MESH_FILE} --decomp-method=parmetis"
+
+cmd "python3 ${HOME}/exawind/source/exawind-cases/tools/reorder_file.py ${SLURM_JOB_NUM_NODES}"
+cmd "mv exawind.reorder_file frontier_exawind.reorder_file"
+AWIND_RANKS=$((${SLURM_JOB_NUM_NODES}*${AWIND_RANK_PER_NODE}))
+NWIND_RANKS=$((${SLURM_JOB_NUM_NODES}*${NWIND_RANK_PER_NODE}))
+TOTAL_RANKS=$((${SLURM_JOB_NUM_NODES}*(${AWIND_RANK_PER_NODE}+${NWIND_RANK_PER_NODE})))
+
+cmd "export MPICH_RANK_REORDER_METHOD=3"
+cmd "export MPICH_RANK_REORDER_FILE=frontier_exawind.reorder_file"
+
+cmd "srun -N${SLURM_JOB_NUM_NODES} -n${TOTAL_RANKS} --gpus-per-node=8 --gpu-bind=closest exawind --awind ${AWIND_RANKS} --nwind ${NWIND_RANKS} nrel5mw.yaml"

exawind/NREL_5MW_Turbine/flexible/input_files/frontier_exawind.reorder_file

@@ -0,0 +1 @@
+256,263,270,277,284,291,298,305,257,264,271,278,285,292,299,306,258,265,272,279,286,293,300,307,259,266,273,280,287,294,301,308,260,267,274,281,288,295,302,309,261,268,275,282,289,296,303,310,262,269,276,283,290,297,304,311,0,1,2,3,4,5,6,7,312,319,326,333,340,347,354,361,313,320,327,334,341,348,355,362,314,321,328,335,342,349,356,363,315,322,329,336,343,350,357,364,316,323,330,337,344,351,358,365,317,324,331,338,345,352,359,366,318,325,332,339,346,353,360,367,8,9,10,11,12,13,14,15,368,375,382,389,396,403,410,417,369,376,383,390,397,404,411,418,370,377,384,391,398,405,412,419,371,378,385,392,399,406,413,420,372,379,386,393,400,407,414,421,373,380,387,394,401,408,415,422,374,381,388,395,402,409,416,423,16,17,18,19,20,21,22,23,424,431,438,445,452,459,466,473,425,432,439,446,453,460,467,474,426,433,440,447,454,461,468,475,427,434,441,448,455,462,469,476,428,435,442,449,456,463,470,477,429,436,443,450,457,464,471,478,430,437,444,451,458,465,472,479,24,25,26,27,28,29,30,31,480,487,494,501,508,515,522,529,481,488,495,502,509,516,523,530,482,489,496,503,510,517,524,531,483,490,497,504,511,518,525,532,484,491,498,505,512,519,526,533,485,492,499,506,513,520,527,534,486,493,500,507,514,521,528,535,32,33,34,35,36,37,38,39,536,543,550,557,564,571,578,585,537,544,551,558,565,572,579,586,538,545,552,559,566,573,580,587,539,546,553,560,567,574,581,588,540,547,554,561,568,575,582,589,541,548,555,562,569,576,583,590,542,549,556,563,570,577,584,591,40,41,42,43,44,45,46,47,592,599,606,613,620,627,634,641,593,600,607,614,621,628,635,642,594,601,608,615,622,629,636,643,595,602,609,616,623,630,637,644,596,603,610,617,624,631,638,645,597,604,611,618,625,632,639,646,598,605,612,619,626,633,640,647,48,49,50,51,52,53,54,55,648,655,662,669,676,683,690,697,649,656,663,670,677,684,691,698,650,657,664,671,678,685,692,699,651,658,665,672,679,686,693,700,652,659,666,673,680,687,694,701,653,660,667,674,681,688,695,702,654,661,668,675,682,689,696,703,56,57,58,59,60,61,62,63,704,711,718,725,732,739,746,753,705,712,719,726,733,740,747,754,706,713,720,727,734,741,748,755,707,714,721,728,735,742,749,756,708,715,722,729,736,743,750,757,709,716,723,730,737,744,751,758,710,717,724,731,738,745,752,759,64,65,66,67,68,69,70,71,760,767,774,781,788,795,802,809,761,768,775,782,789,796,803,810,762,769,776,783,790,797,804,811,763,770,777,784,791,798,805,812,764,771,778,785,792,799,806,813,765,772,779,786,793,800,807,814,766,773,780,787,794,801,808,815,72,73,74,75,76,77,78,79,816,823,830,837,844,851,858,865,817,824,831,838,845,852,859,866,818,825,832,839,846,853,860,867,819,826,833,840,847,854,861,868,820,827,834,841,848,855,862,869,821,828,835,842,849,856,863,870,822,829,836,843,850,857,864,871,80,81,82,83,84,85,86,87,872,879,886,893,900,907,914,921,873,880,887,894,901,908,915,922,874,881,888,895,902,909,916,923,875,882,889,896,903,910,917,924,876,883,890,897,904,911,918,925,877,884,891,898,905,912,919,926,878,885,892,899,906,913,920,927,88,89,90,91,92,93,94,95,928,935,942,949,956,963,970,977,929,936,943,950,957,964,971,978,930,937,944,951,958,965,972,979,931,938,945,952,959,966,973,980,932,939,946,953,960,967,974,981,933,940,947,954,961,968,975,982,934,941,948,955,962,969,976,983,96,97,98,99,100,101,102,103,984,991,998,1005,1012,1019,1026,1033,985,992,999,1006,1013,1020,1027,1034,986,993,1000,1007,1014,1021,1028,1035,987,994,1001,1008,1015,1022,1029,1036,988,995,1002,1009,1016,1023,1030,1037,989,996,1003,1010,1017,1024,1031,1038,990,997,1004,1011,1018,1025,1032,1039,104,105,106,107,108,109,110,111,1040,1047,1054,1061,1068,1075,1082,1089,1041,1048,1055,1062,1069,1076,1083,1090,1042,1049,1056,1063,1070,1077,1084,1091,1043,1050,1057,1064,1071,1078,1085,1092,1044,1051,1058,1065,1072,1079,1086,1093,1045,1052,1059,1066,1073,1080,1087,1094,1046,1053,1060,1067,1074,1081,1088,1095,112,113,114,115,116,117,118,119,1096,1103,1110,1117,1124,1131,1138,1145,1097,1104,1111,1118,1125,1132,1139,1146,1098,1105,1112,1119,1126,1133,1140,1147,1099,1106,1113,1120,1127,1134,1141,1148,1100,1107,1114,1121,1128,1135,1142,1149,1101,1108,1115,1122,1129,1136,1143,1150,1102,1109,1116,1123,1130,1137,1144,1151,120,121,122,123,124,125,126,127,1152,1159,1166,1173,1180,1187,1194,1201,1153,1160,1167,1174,1181,1188,1195,1202,1154,1161,1168,1175,1182,1189,1196,1203,1155,1162,1169,1176,1183,1190,1197,1204,1156,1163,1170,1177,1184,1191,1198,1205,1157,1164,1171,1178,1185,1192,1199,1206,1158,1165,1172,1179,1186,1193,1200,1207,128,129,130,131,132,133,134,135,1208,1215,1222,1229,1236,1243,1250,1257,1209,1216,1223,1230,1237,1244,1251,1258,1210,1217,1224,1231,1238,1245,1252,1259,1211,1218,1225,1232,1239,1246,1253,1260,1212,1219,1226,1233,1240,1247,1254,1261,1213,1220,1227,1234,1241,1248,1255,1262,1214,1221,1228,1235,1242,1249,1256,1263,136,137,138,139,140,141,142,143,1264,1271,1278,1285,1292,1299,1306,1313,1265,1272,1279,1286,1293,1300,1307,1314,1266,1273,1280,1287,1294,1301,1308,1315,1267,1274,1281,1288,1295,1302,1309,1316,1268,1275,1282,1289,1296,1303,1310,1317,1269,1276,1283,1290,1297,1304,1311,1318,1270,1277,1284,1291,1298,1305,1312,1319,144,145,146,147,148,149,150,151,1320,1327,1334,1341,1348,1355,1362,1369,1321,1328,1335,1342,1349,1356,1363,1370,1322,1329,1336,1343,1350,1357,1364,1371,1323,1330,1337,1344,1351,1358,1365,1372,1324,1331,1338,1345,1352,1359,1366,1373,1325,1332,1339,1346,1353,1360,1367,1374,1326,1333,1340,1347,1354,1361,1368,1375,152,153,154,155,156,157,158,159,1376,1383,1390,1397,1404,1411,1418,1425,1377,1384,1391,1398,1405,1412,1419,1426,1378,1385,1392,1399,1406,1413,1420,1427,1379,1386,1393,1400,1407,1414,1421,1428,1380,1387,1394,1401,1408,1415,1422,1429,1381,1388,1395,1402,1409,1416,1423,1430,1382,1389,1396,1403,1410,1417,1424,1431,160,161,162,163,164,165,166,167,1432,1439,1446,1453,1460,1467,1474,1481,1433,1440,1447,1454,1461,1468,1475,1482,1434,1441,1448,1455,1462,1469,1476,1483,1435,1442,1449,1456,1463,1470,1477,1484,1436,1443,1450,1457,1464,1471,1478,1485,1437,1444,1451,1458,1465,1472,1479,1486,1438,1445,1452,1459,1466,1473,1480,1487,168,169,170,171,172,173,174,175,1488,1495,1502,1509,1516,1523,1530,1537,1489,1496,1503,1510,1517,1524,1531,1538,1490,1497,1504,1511,1518,1525,1532,1539,1491,1498,1505,1512,1519,1526,1533,1540,1492,1499,1506,1513,1520,1527,1534,1541,1493,1500,1507,1514,1521,1528,1535,1542,1494,1501,1508,1515,1522,1529,1536,1543,176,177,178,179,180,181,182,183,1544,1551,1558,1565,1572,1579,1586,1593,1545,1552,1559,1566,1573,1580,1587,1594,1546,1553,1560,1567,1574,1581,1588,1595,1547,1554,1561,1568,1575,1582,1589,1596,1548,1555,1562,1569,1576,1583,1590,1597,1549,1556,1563,1570,1577,1584,1591,1598,1550,1557,1564,1571,1578,1585,1592,1599,184,185,186,187,188,189,190,191,1600,1607,1614,1621,1628,1635,1642,1649,1601,1608,1615,1622,1629,1636,1643,1650,1602,1609,1616,1623,1630,1637,1644,1651,1603,1610,1617,1624,1631,1638,1645,1652,1604,1611,1618,1625,1632,1639,1646,1653,1605,1612,1619,1626,1633,1640,1647,1654,1606,1613,1620,1627,1634,1641,1648,1655,192,193,194,195,196,197,198,199,1656,1663,1670,1677,1684,1691,1698,1705,1657,1664,1671,1678,1685,1692,1699,1706,1658,1665,1672,1679,1686,1693,1700,1707,1659,1666,1673,1680,1687,1694,1701,1708,1660,1667,1674,1681,1688,1695,1702,1709,1661,1668,1675,1682,1689,1696,1703,1710,1662,1669,1676,1683,1690,1697,1704,1711,200,201,202,203,204,205,206,207,1712,1719,1726,1733,1740,1747,1754,1761,1713,1720,1727,1734,1741,1748,1755,1762,1714,1721,1728,1735,1742,1749,1756,1763,1715,1722,1729,1736,1743,1750,1757,1764,1716,1723,1730,1737,1744,1751,1758,1765,1717,1724,1731,1738,1745,1752,1759,1766,1718,1725,1732,1739,1746,1753,1760,1767,208,209,210,211,212,213,214,215,1768,1775,1782,1789,1796,1803,1810,1817,1769,1776,1783,1790,1797,1804,1811,1818,1770,1777,1784,1791,1798,1805,1812,1819,1771,1778,1785,1792,1799,1806,1813,1820,1772,1779,1786,1793,1800,1807,1814,1821,1773,1780,1787,1794,1801,1808,1815,1822,1774,1781,1788,1795,1802,1809,1816,1823,216,217,218,219,220,221,222,223,1824,1831,1838,1845,1852,1859,1866,1873,1825,1832,1839,1846,1853,1860,1867,1874,1826,1833,1840,1847,1854,1861,1868,1875,1827,1834,1841,1848,1855,1862,1869,1876,1828,1835,1842,1849,1856,1863,1870,1877,1829,1836,1843,1850,1857,1864,1871,1878,1830,1837,1844,1851,1858,1865,1872,1879,224,225,226,227,228,229,230,231,1880,1887,1894,1901,1908,1915,1922,1929,1881,1888,1895,1902,1909,1916,1923,1930,1882,1889,1896,1903,1910,1917,1924,1931,1883,1890,1897,1904,1911,1918,1925,1932,1884,1891,1898,1905,1912,1919,1926,1933,1885,1892,1899,1906,1913,1920,1927,1934,1886,1893,1900,1907,1914,1921,1928,1935,232,233,234,235,236,237,238,239,1936,1943,1950,1957,1964,1971,1978,1985,1937,1944,1951,1958,1965,1972,1979,1986,1938,1945,1952,1959,1966,1973,1980,1987,1939,1946,1953,1960,1967,1974,1981,1988,1940,1947,1954,1961,1968,1975,1982,1989,1941,1948,1955,1962,1969,1976,1983,1990,1942,1949,1956,1963,1970,1977,1984,1991,240,241,242,243,244,245,246,247,1992,1999,2006,2013,2020,2027,2034,2041,1993,2000,2007,2014,2021,2028,2035,2042,1994,2001,2008,2015,2022,2029,2036,2043,1995,2002,2009,2016,2023,2030,2037,2044,1996,2003,2010,2017,2024,2031,2038,2045,1997,2004,2011,2018,2025,2032,2039,2046,1998,2005,2012,2019,2026,2033,2040,2047,248,249,250,251,252,253,254,255

exawind/NREL_5MW_Turbine/flexible/input_files/nrel5mw_amr.inp

@@ -9,7 +9,7 @@ incflo.physics                           = ABL                 # List of physics
 incflo.verbose                           = 0                   
 io.plot_file                             = out/plt  
 io.check_file                            = rst/chk                 
-io.restart_file                          = /gpfs/lcheung/HFM/exawind-benchmarks/Neutral_ABL/chk30000
+io.restart_file                          = /lustre/orion/cfd162/world-shared/lcheung/HFM/exawind-benchmarks/amr-wind/atmospheric_boundary_layer/convective_abl_nrel5mw/runfiles/chk30000
 io.KE_int                                = -1 
 
 incflo.use_godunov                       = true                
@@ -52,7 +52,7 @@ ABL.stats_output_frequency               = 1
 ABL.stats_output_format                  = netcdf              
 ABL.tendency_forcing                     = false               
 ABL.bndry_io_mode                        = 1                   
-ABL.bndry_file                           = /gpfs/lcheung/HFM/exawind-benchmarks/Neutral_ABL/bndry_file
+ABL.bndry_file                           = /lustre/orion/cfd162/world-shared/lcheung/HFM/exawind-benchmarks/amr-wind/atmospheric_boundary_layer/convective_abl_nrel5mw/runfiles/bndry_file
 ABL.bndry_planes                         = xlo ylo             
 ABL.bndry_output_start_time              = 15000.0             
 ABL.bndry_var_names                      = velocity temperature tke
@@ -242,14 +242,14 @@ nodal_proj.mg_atol                       = 1e-10
 mac_proj.mg_rtol                         = 1e-06               
 mac_proj.mg_atol                         = 1e-10  
 
-nodal_proj.max_coarsening_level          = 0                   
+# nodal_proj.max_coarsening_level          = 0                   
 nodal_proj.num_pre_smooth                = 10                  
 nodal_proj.num_post_smooth               = 10                  
 nodal_proj.bottom_rtol                   = 1.0e-9              
 nodal_proj.bottom_atol                   = 1.0e-12
 nodal_proj.bottom_verbose                = 1             
 mac_proj.verbose                         = 0                   
-mac_proj.max_coarsening_level            = 0                   
+# mac_proj.max_coarsening_level            = 0                   
 mac_proj.do_nsolve                       = true                
 mac_proj.bottom_verbose                  = 1                   
 mac_proj.bottom_rtol                     = 1.0e-11             

exawind/NREL_5MW_Turbine/flexible/input_files/nrel5mw_nalu.yaml

@@ -70,7 +70,7 @@ linear_solvers:
 
 realms:
 - name: realm_1
-  mesh: /pscratch/ndeveld/hfm-2025-q1/mesh/combine_split/split_tower_and_blades.exo
+  mesh: /lustre/orion/cfd162/proj-shared/marchdf/exawind/source/exawind-benchmarks/exawind/NREL_5MW_Turbine/flexible/mesh/split_tower_and_blades.exo
   #automatic_decomposition_type: rcb
   #rebalance_mesh: yes
   #stk_rebalance_method: parmetis

exawind/NREL_5MW_Turbine/flexible/input_files_4turb/5MW_Baseline/AeroData/Cylinder1.dat

@@ -0,0 +1,19 @@
+Round root section with a Cd of 0.50
+Made by Jason Jonkman
+   1        Number of airfoil tables in this file
+   0.0      Table ID parameter
+   0.0      Stall angle (deg)
+   0.0      No longer used, enter zero
+   0.0      No longer used, enter zero
+   0.0      No longer used, enter zero
+   0.0      Zero Cn angle of attack (deg)
+   0.0      Cn slope for zero lift (dimensionless)
+   0.0      Cn extrapolated to value at positive stall angle of attack
+   0.0      Cn at stall value for negative angle of attack
+   0.0      Angle of attack for minimum CD (deg)
+   0.50     Minimum CD value
+-180.00    0.000   0.5000   0.000
+   0.00    0.000   0.5000   0.000
+ 180.00    0.000   0.5000   0.000
+
+

exawind/NREL_5MW_Turbine/flexible/input_files_4turb/5MW_Baseline/AeroData/Cylinder2.dat

@@ -0,0 +1,19 @@
+Round root section with a Cd of 0.35
+Made by Jason Jonkman
+   1        Number of airfoil tables in this file
+   0.0      Table ID parameter
+   0.0      Stall angle (deg)
+   0.0      No longer used, enter zero
+   0.0      No longer used, enter zero
+   0.0      No longer used, enter zero
+   0.0      Zero Cn angle of attack (deg)
+   0.0      Cn slope for zero lift (dimensionless)
+   0.0      Cn extrapolated to value at positive stall angle of attack
+   0.0      Cn at stall value for negative angle of attack
+   0.0      Angle of attack for minimum CD (deg)
+   0.35     Minimum CD value
+-180.00    0.000   0.3500   0.000
+   0.00    0.000   0.3500   0.000
+ 180.00    0.000   0.3500   0.000
+
+