HILA
Loading...
Searching...
No Matches
setup_layout_generic.cpp
1/// Setup layout does the node division. This version
2/// first tries an even distribution, with equally sized
3/// nodes, and if that fails allows slightly different
4/// node sizes.
5
6#include "plumbing/defs.h"
7#include "plumbing/lattice.h"
8
9/***************************************************************/
10
11/* number of primes to be used in factorization */
12#define NPRIMES 12
13const static int prime[NPRIMES] = {2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37};
14
15/* Set up now squaresize and nsquares - arrays
16 * Print info to outf as we proceed
17 */
18
19void lattice_struct::setup_layout() {
20 int nfactors[NPRIMES];
21 CoordinateVector nodesiz;
22
23 hila::print_dashed_line();
24 hila::out0 << "LAYOUT: lattice size ";
25 foralldir(d) {
26 if (d != 0)
27 hila::out0 << " x ";
28 hila::out0 << l_size[d];
29 }
30 hila::out0 << " = " << l_volume << " sites\n";
31 hila::out0 << "Dividing to " << hila::number_of_nodes() << " nodes\n";
32
33 // foralldir(d) if (l_size[d] % 2 != 0) {
34 // hila::out0 << "Lattice must be even to all directions (odd size:TODO)\n";
35 // hila::finishrun();
36 // }
37
38 // Factorize the node number in primes
39 // These factors must be used in slicing the lattice!
40 int nn = hila::number_of_nodes();
41
42 int i = nn;
43 for (int n = 0; n < NPRIMES; n++) {
44 nfactors[n] = 0;
45 while (i % prime[n] == 0) {
46 i /= prime[n];
47 nfactors[n]++;
48 }
49 }
50 if (i != 1) {
51 hila::out0 << "Cannot factorize " << nn << " nodes with primes up to " << prime[NPRIMES - 1]
52 << '\n';
54 }
55
56 // strategy: try to increase the box size to one of the directions until rem = 0
57 // find the optimal direction to do it
58 // Use simple heuristic: take the dim with the least amount of added "ghost sites"
59
60 CoordinateVector nsize;
61 int64_t ghosts[NDIM]; // int is too small
62
63 foralldir(d) {
64 int64_t cosize = l_volume / l_size[d];
65 int64_t n = l_size[d];
66 while ((n * cosize) % nn != 0)
67 n++; // virtual size can be odd
68 // now nsize is the new would-be size
69 ghosts[d] = (n - l_size[d]) * cosize;
70 nsize[d] = n;
71 }
72
73 int mdir = 0;
74 bool secondtime = false;
75 do {
76 // try the division a couple of times, if the 1st fails
77
78 foralldir(j) if (ghosts[mdir] > ghosts[j]) mdir = j;
79 // mdir is the direction where we do uneven division (if done)
80 // hila::out0 << "MDIR " << mdir << " ghosts mdir " << ghosts[mdir] << " nsize " <<
81 // nsize[mdir] << '\n';
82
83 foralldir(i) {
84 nodesiz[i] = (i == mdir) ? nsize[i] : l_size[i]; // start with ghosted lattice size
85 nodes.n_divisions[i] = 1;
86 }
87
88 for (int n = NPRIMES - 1; n >= 0; n--)
89 for (int i = 0; i < nfactors[n]; i++) {
90 // figure out which direction to divide -- start from the largest prime,
91 // because we don't want this to be last divisor! (would probably wind up
92 // with size 1)
93
94 // find largest divisible dimension of h-cubes - start from last, because
95 int msize = 1, dir;
96 for (dir = 0; dir < NDIM; dir++)
97 if (nodesiz[dir] > msize && nodesiz[dir] % prime[n] == 0)
98 msize = nodesiz[dir];
99
100 // if one direction with largest dimension has already been
101 // divided, divide it again. Otherwise divide first direction
102 // with largest dimension.
103
104 // Switch here to first divide along t-direction, in
105 // order to
106 // a) minimize spatial blocks
107 // b) In sf t-division is cheaper (1 non-communicating slice)
108
109 for (dir = NDIM - 1; dir >= 0; dir--)
110 if (nodesiz[dir] == msize && nodes.n_divisions[dir] > 1 &&
111 nodesiz[dir] % prime[n] == 0)
112 break;
113
114 // If not previously sliced, take one direction to slice
115 if (dir < 0)
116 for (dir = NDIM - 1; dir >= 0; dir--)
117 if (nodesiz[dir] == msize && nodesiz[dir] % prime[n] == 0)
118 break;
119
120 if (dir < 0) {
121 // This cannot happen
122 hila::out0 << "CANNOT HAPPEN! in setup_layout_generic.c\n";
124 }
125
126 // Now slice it
127 nodesiz[dir] /= prime[n];
128 nodes.n_divisions[dir] *= prime[n];
129 }
130
131 // now check that the div makes sens
132 bool fail = false;
133 foralldir(dir) if (nodesiz[dir] < 2) fail = true; // don't allow nodes of size 1
134 if (fail && !secondtime) {
135 secondtime = true;
136 ghosts[mdir] =
137 (1ULL << 62); // this short-circuits direction mdir, some other taken next
138 } else if (fail) {
139 hila::out0 << "Could not successfully lay out the lattice with "
140 << hila::number_of_nodes() << " nodes\n";
142 }
143
144 } while (secondtime);
145
146 // set up struct nodes variables
147 nodes.number = hila::number_of_nodes();
149
150 // Now division done - check how good it is
151 int ghost_slices = nsize[mdir] - l_size[mdir];
152 if (ghost_slices > 0) {
153 hila::out0 << "\nUsing uneven node division to direction " << mdir << ":\n";
154 hila::out0 << "Lengths: " << nodes.n_divisions[mdir] - ghost_slices << " * ("
155 << nodesiz[mdir] << " sites) + " << ghost_slices << " * (" << nodesiz[mdir] - 1
156 << " sites)\n";
157 hila::out0 << "Divisions: ";
158 for (int i = 0; i < nodes.n_divisions[mdir]; i++) {
159 if (i > 0)
160 hila::out0 << " - ";
161 hila::out0 << nodes.divisors[mdir][i + 1] - nodes.divisors[mdir][i];
162 }
163 hila::out0 << "\nFilling efficiency: " << (100.0 * l_size[mdir]) / nsize[mdir] << "%\n";
164
165 if (ghost_slices > nodes.n_divisions[mdir] / 2)
166 hila::out0 << "NOTE: number of smaller nodes > large nodes \n";
167 }
168
169 // this was hila::number_of_nodes() > 1
170 if (1) {
171 hila::out0 << "\nSites on node: ";
172 foralldir(dir) {
173 if (dir > 0)
174 hila::out0 << " x ";
175 if (dir == mdir && ghost_slices > 0)
176 hila::out0 << '(' << nodesiz[dir] - 1 << '-' << nodesiz[dir] << ')';
177 else
178 hila::out0 << nodesiz[dir];
179 }
180 int ns = 1;
181 foralldir(dir) ns *= nodesiz[dir];
182 if (ghost_slices > 0) {
183 int ns2 = ns * (nodesiz[mdir] - 1) / nodesiz[mdir];
184 hila::out0 << " = " << ns2 << " - " << ns << '\n';
185 } else {
186 hila::out0 << " = " << ns << '\n';
187 }
188
189 hila::out0 << "Processor layout: ";
190 foralldir(dir) {
191 if (dir > 0)
192 hila::out0 << " x ";
193 hila::out0 << nodes.n_divisions[dir];
194 }
195 hila::out0 << " = " << hila::number_of_nodes() << " nodes\n";
196 }
197
198 // For MPI, remap the nodes for periodic torus
199 // in the desired manner
200 // we have at least 2 options:
201 // map_node_layout_trivial.c
202 // map_node_layout_block2.c - for 2^n n.n. blocks
203
204 nodes.create_remap();
205
206
207 hila::print_dashed_line();
208}
void setup_node_divisors()
Definition lattice.cpp:123
#define foralldir(d)
Macro to loop over (all) Direction(s)
Definition coordinates.h:80
This file defines all includes for HILA.
int number_of_nodes()
how many nodes there are
Definition com_mpi.cpp:248
std::ostream out0
This writes output only from main process (node 0)
void finishrun()
Normal, controlled exit - all nodes must call this. Prints timing information and information about c...