problem.cc
Go to the documentation of this file.
1 // LIC// ====================================================================
2 // LIC// This file forms part of oomph-lib, the object-oriented,
3 // LIC// multi-physics finite-element library, available
4 // LIC// at http://www.oomph-lib.org.
5 // LIC//
6 // LIC// Copyright (C) 2006-2024 Matthias Heil and Andrew Hazel
7 // LIC//
8 // LIC// This library is free software; you can redistribute it and/or
9 // LIC// modify it under the terms of the GNU Lesser General Public
10 // LIC// License as published by the Free Software Foundation; either
11 // LIC// version 2.1 of the License, or (at your option) any later version.
12 // LIC//
13 // LIC// This library is distributed in the hope that it will be useful,
14 // LIC// but WITHOUT ANY WARRANTY; without even the implied warranty of
15 // LIC// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 // LIC// Lesser General Public License for more details.
17 // LIC//
18 // LIC// You should have received a copy of the GNU Lesser General Public
19 // LIC// License along with this library; if not, write to the Free Software
20 // LIC// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
21 // LIC// 02110-1301 USA.
22 // LIC//
23 // LIC// The authors may be contacted at oomph-lib@maths.man.ac.uk.
24 // LIC//
25 // LIC//====================================================================
26 
27 #ifdef OOMPH_HAS_MPI
28 #include "mpi.h"
29 #endif
30 
31 #include <list>
32 #include <algorithm>
33 #include <string>
34 
35 #include "oomph_utilities.h"
36 #include "problem.h"
37 #include "timesteppers.h"
38 #include "explicit_timesteppers.h"
40 #include "refineable_mesh.h"
41 #include "triangle_mesh.h"
42 #include "linear_solver.h"
43 #include "eigen_solver.h"
44 #include "assembly_handler.h"
45 #include "dg_elements.h"
46 #include "partitioning.h"
47 #include "spines.h"
48 
49 // Include to fill in additional_setup_shared_node_scheme() function
51 
52 
53 namespace oomph
54 {
55  /// ///////////////////////////////////////////////////////////////
56  // Non-inline functions for the problem class
57  /// ///////////////////////////////////////////////////////////////
58 
59  //=================================================================
60  /// The continuation timestepper object
61  //=================================================================
62  ContinuationStorageScheme Problem::Continuation_time_stepper;
63 
64  //================================================================
65  /// Constructor: Allocate space for one time stepper
66  /// and set all pointers to NULL and set defaults for all
67  /// parameters.
68  //===============================================================
70  : Mesh_pt(0),
71  Time_pt(0),
72  Explicit_time_stepper_pt(0),
73  Saved_dof_pt(0),
74  Default_set_initial_condition_called(false),
75  Use_globally_convergent_newton_method(false),
76  Empty_actions_before_read_unstructured_meshes_has_been_called(false),
77  Empty_actions_after_read_unstructured_meshes_has_been_called(false),
78  Store_local_dof_pt_in_elements(false),
79  Calculate_hessian_products_analytic(false),
80 #ifdef OOMPH_HAS_MPI
81  Doc_imbalance_in_parallel_assembly(false),
82  Use_default_partition_in_load_balance(false),
83  Must_recompute_load_balance_for_assembly(true),
84  Halo_scheme_pt(0),
85 #endif
86  Relaxation_factor(1.0),
87  Newton_solver_tolerance(1.0e-8),
89  Nnewton_iter_taken(0),
90  Max_residuals(10.0),
91  Time_adaptive_newton_crash_on_solve_fail(false),
92  Jacobian_reuse_is_enabled(false),
93  Jacobian_has_been_computed(false),
94  Problem_is_nonlinear(true),
95  Pause_at_end_of_sparse_assembly(false),
96  Doc_time_in_distribute(false),
97  Sparse_assembly_method(Perform_assembly_using_vectors_of_pairs),
98  Sparse_assemble_with_arrays_initial_allocation(400),
99  Sparse_assemble_with_arrays_allocation_increment(150),
100  Numerical_zero_for_sparse_assembly(0.0),
101  FD_step_used_in_get_hessian_vector_products(1.0e-8),
102  Mass_matrix_reuse_is_enabled(false),
103  Mass_matrix_has_been_computed(false),
104  Discontinuous_element_formulation(false),
105  Minimum_dt(1.0e-12),
106  Maximum_dt(1.0e12),
107  DTSF_max_increase(4.0),
108  DTSF_min_decrease(0.8),
109  Target_error_safety_factor(1.0),
110  Minimum_dt_but_still_proceed(-1.0),
111  Scale_arc_length(true),
112  Desired_proportion_of_arc_length(0.5),
113  Theta_squared(1.0),
114  Sign_of_jacobian(0),
115  Continuation_direction(1.0),
116  Parameter_derivative(1.0),
117  Parameter_current(0.0),
118  Use_continuation_timestepper(false),
119  Dof_derivative_offset(1),
120  Dof_current_offset(2),
121  Ds_current(0.0),
122  Desired_newton_iterations_ds(5),
123  Minimum_ds(1.0e-10),
124  Bifurcation_detection(false),
125  Bisect_to_find_bifurcation(false),
126  First_jacobian_sign_change(false),
127  Arc_length_step_taken(false),
128  Use_finite_differences_for_continuation_derivatives(false),
129 #ifdef OOMPH_HAS_MPI
130  Dist_problem_matrix_distribution(Uniform_matrix_distribution),
131  Parallel_sparse_assemble_previous_allocation(0),
132  Problem_has_been_distributed(false),
133  Bypass_increase_in_dof_check_during_pruning(false),
134  Max_permitted_error_for_halo_check(1.0e-14),
135 #endif
136  Shut_up_in_newton_solve(false),
137  Always_take_one_newton_step(false),
138  Timestep_reduction_factor_after_nonconvergence(0.5),
139  Keep_temporal_error_below_tolerance(true)
140  {
142 
143  /// Setup terminate helper
145 
146  // By default no submeshes:
147  Sub_mesh_pt.resize(0);
148  // No timesteppers
149  Time_stepper_pt.resize(0);
150 
151  // Set the linear solvers, eigensolver and assembly handler
154 
156 
158 
159  // setup the communicator
160 #ifdef OOMPH_HAS_MPI
162  {
164  }
165  else
166  {
168  }
169 #else
171 #endif
172 
173  // just create an empty linear algebra distribution for the
174  // DOFs
175  // this is setup when assign_eqn_numbers(...) is called.
177  }
178 
179  //================================================================
180  /// Destructor to clean up memory
181  //================================================================
183  {
184  // Delete the memory assigned for the global time
185  // (it's created on the fly in Problem::add_time_stepper_pt()
186  // so we are entitled to delete it.
187  if (Time_pt != 0)
188  {
189  delete Time_pt;
190  Time_pt = 0;
191  }
192 
193  // We're not using the default linear solver,
194  // somebody else must have built it, so that person
195  // must be in charge of killing it.
196  // We can safely delete the defaults, however
198 
201  delete Communicator_pt;
202  delete Dof_distribution_pt;
203 
204  // Delete any copies of the problem that have been created for
205  // use in adaptive bifurcation tracking.
206  // ALH: This will eventually go
207  unsigned n_copies = Copy_of_problem_pt.size();
208  for (unsigned c = 0; c < n_copies; c++)
209  {
210  delete Copy_of_problem_pt[c];
211  }
212 
213  // if this problem has sub meshes then we must delete the Mesh_pt
214  if (Sub_mesh_pt.size() != 0)
215  {
217  delete Mesh_pt;
218  }
219 
220  // Since we called the TerminateHelper setup function in the constructor,
221  // we need to delete anything that was dynamically allocated (as it's
222  // just a namespace and so doesn't have it's own destructor) in the function
224  }
225 
226  //=================================================================
227  /// Setup the count vector that records how many elements contribute
228  /// to each degree of freedom. Returns the total number of elements
229  /// in the problem
230  //=================================================================
232  {
233  // Now set the element counter to have the current Dof distribution
235  // We need to use the halo scheme (assuming it has been setup)
236 #ifdef OOMPH_HAS_MPI
238 #endif
239 
240  // Loop over the elements and count the entries
241  // and number of (non-halo) elements
242  const unsigned n_element = this->mesh_pt()->nelement();
243  unsigned n_non_halo_element_local = 0;
244  for (unsigned e = 0; e < n_element; e++)
245  {
246  GeneralisedElement* elem_pt = this->mesh_pt()->element_pt(e);
247 #ifdef OOMPH_HAS_MPI
248  // Ignore halo elements
249  if (!elem_pt->is_halo())
250  {
251 #endif
252  // Increment the number of non halo elements
253  ++n_non_halo_element_local;
254  // Now count the number of times the element contributes to a value
255  // using the current assembly handler
256  unsigned n_var = this->Assembly_handler_pt->ndof(elem_pt);
257  for (unsigned n = 0; n < n_var; n++)
258  {
260  this->Assembly_handler_pt->eqn_number(elem_pt, n));
261  }
262 #ifdef OOMPH_HAS_MPI
263  }
264 #endif
265  }
266 
267  // Storage for the total number of elements
268  unsigned Nelement = 0;
269 
270  // Add together all the counts if we are in parallel
271 #ifdef OOMPH_HAS_MPI
273 
274  // If distributed, find the total number of elements in the problem
276  {
277  // Need to gather the total number of non halo elements
278  MPI_Allreduce(&n_non_halo_element_local,
279  &Nelement,
280  1,
281  MPI_UNSIGNED,
282  MPI_SUM,
283  this->communicator_pt()->mpi_comm());
284  }
285  // Otherwise the total number is the same on each processor
286  else
287 #endif
288  {
289  Nelement = n_non_halo_element_local;
290  }
291 
292  return Nelement;
293  }
294 
295 
296  //==================================================================
297  /// Build new LinearAlgebraDistribution. Note: you're in charge of
298  /// deleting it!
299  //==================================================================
301  LinearAlgebraDistribution*& dist_pt)
302  {
303  // Find the number of rows
304  const unsigned nrow = this->ndof();
305 
306 #ifdef OOMPH_HAS_MPI
307 
308  unsigned nproc = Communicator_pt->nproc();
309 
310  // if problem is only one one processor assemble non-distributed
311  // distribution
312  if (nproc == 1)
313  {
314  dist_pt = new LinearAlgebraDistribution(Communicator_pt, nrow, false);
315  }
316  // if the problem is not distributed then assemble the jacobian with
317  // a uniform distributed distribution
319  {
320  dist_pt = new LinearAlgebraDistribution(Communicator_pt, nrow, true);
321  }
322  // otherwise the problem is a distributed problem
323  else
324  {
326  {
328 
329  dist_pt = new LinearAlgebraDistribution(Communicator_pt, nrow, true);
330  break;
331 
333 
335  break;
336 
338 
339  // Put in its own scope to avoid warnings about "local" variables
340  {
341  LinearAlgebraDistribution* uniform_dist_pt =
343  bool use_problem_dist = true;
344  for (unsigned p = 0; p < nproc; p++)
345  {
346  // hierher Andrew: what's the logic behind this?
347  if ((double)Dof_distribution_pt->nrow_local(p) >
348  ((double)uniform_dist_pt->nrow_local(p)) * 1.1)
349  {
350  use_problem_dist = false;
351  }
352  }
353  if (use_problem_dist)
354  {
356  }
357  else
358  {
359  dist_pt = new LinearAlgebraDistribution(uniform_dist_pt);
360  }
361  delete uniform_dist_pt;
362  }
363  break;
364 
365  default:
366 
367  std::ostringstream error_stream;
368  error_stream << "Never get here. Dist_problem_matrix_distribution = "
369  << Dist_problem_matrix_distribution << std::endl;
370  throw OomphLibError(error_stream.str(),
371  OOMPH_CURRENT_FUNCTION,
372  OOMPH_EXCEPTION_LOCATION);
373  break;
374  }
375  }
376 #else
377  dist_pt = new LinearAlgebraDistribution(Communicator_pt, nrow, false);
378 #endif
379  }
380 
381 
382 #ifdef OOMPH_HAS_MPI
383 
384  //==================================================================
385  /// Setup the halo scheme for the degrees of freedom
386  //==================================================================
388  {
389  // Find the number of elements stored on this processor
390  const unsigned n_element = this->mesh_pt()->nelement();
391 
392  // Work out the all global equations to which this processor
393  // contributes
394  Vector<unsigned> my_eqns;
395  this->get_my_eqns(this->Assembly_handler_pt, 0, n_element - 1, my_eqns);
396 
397  // Build the halo scheme, based on the equations to which this
398  // processor contributes
400  new DoubleVectorHaloScheme(this->Dof_distribution_pt, my_eqns);
401 
402  // Find pointers to all the halo dofs
403  // There may be more of these than required by my_eqns
404  //(but should not be less)
405  std::map<unsigned, double*> halo_data_pt;
406  this->get_all_halo_data(halo_data_pt);
407 
408  // Now setup the Halo_dofs
409  Halo_scheme_pt->setup_halo_dofs(halo_data_pt, this->Halo_dof_pt);
410  }
411 
412  //==================================================================
413  /// Distribute the problem without doc; report stats if required.
414  /// Returns actual partitioning used, e.g. for restart.
415  //==================================================================
416  Vector<unsigned> Problem::distribute(const bool& report_stats)
417  {
418  // Set dummy doc paramemters
419  DocInfo doc_info;
420  doc_info.disable_doc();
421 
422  // Set the sizes of the input and output vectors
423  unsigned n_element = mesh_pt()->nelement();
424  Vector<unsigned> element_partition(n_element, 0);
425 
426  // Distribute and return partitioning
427  return distribute(element_partition, doc_info, report_stats);
428  }
429 
430  //==================================================================
431  /// Distribute the problem according to specified partition.
432  /// If all entries in partitioning vector are zero we use METIS
433  /// to do the partitioning after all.
434  /// Returns actual partitioning used, e.g. for restart.
435  //==================================================================
437  const Vector<unsigned>& element_partition, const bool& report_stats)
438  {
439 #ifdef PARANOID
440  bool has_non_zero_entry = false;
441  unsigned n = element_partition.size();
442  for (unsigned i = 0; i < n; i++)
443  {
444  if (element_partition[i] != 0)
445  {
446  has_non_zero_entry = true;
447  break;
448  }
449  }
450  if (!has_non_zero_entry)
451  {
452  std::ostringstream warn_message;
453  warn_message << "WARNING: All entries in specified partitioning vector \n"
454  << " are zero -- will ignore this and use METIS\n"
455  << " to perform the partitioning\n";
457  warn_message.str(), "Problem::distribute()", OOMPH_EXCEPTION_LOCATION);
458  }
459 #endif
460  // Set dummy doc paramemters
461  DocInfo doc_info;
462  doc_info.disable_doc();
463 
464  // Distribute and return partitioning
465  return distribute(element_partition, doc_info, report_stats);
466  }
467 
468  //==================================================================
469  /// Distribute the problem and doc to specified DocInfo.
470  /// Returns actual partitioning used, e.g. for restart.
471  //==================================================================
473  const bool& report_stats)
474  {
475  // Set the sizes of the input and output vectors
476  unsigned n_element = mesh_pt()->nelement();
477 
478  // Dummy input vector
479  Vector<unsigned> element_partition(n_element, 0);
480 
481  // Distribute and return partitioning
482  return distribute(element_partition, doc_info, report_stats);
483  }
484 
485  //==================================================================
486  /// Distribute the problem according to specified partition.
487  /// (If all entries in partitioning vector are zero we use METIS
488  /// to do the partitioning after all) and doc.
489  /// Returns actual partitioning used, e.g. for restart.
490  //==================================================================
492  const Vector<unsigned>& element_partition,
493  DocInfo& doc_info,
494  const bool& report_stats)
495  {
496  // Storage for number of processors and number of elements in global mesh
497  int n_proc = this->communicator_pt()->nproc();
498  int my_rank = this->communicator_pt()->my_rank();
499  int n_element = mesh_pt()->nelement();
500 
501  // Vector to be returned
502  Vector<unsigned> return_element_domain;
503 
504  // Buffer extreme cases
505  if (n_proc == 1) // single-process job - don't do anything
506  {
507  if (report_stats)
508  {
509  std::ostringstream warn_message;
510  warn_message << "WARNING: You've tried to distribute a problem over\n"
511  << "only one processor: this would make METIS crash.\n"
512  << "Ignoring your request for distribution.\n";
513  OomphLibWarning(warn_message.str(),
514  "Problem::distribute()",
515  OOMPH_EXCEPTION_LOCATION);
516  }
517  }
518  else if (n_proc > n_element) // more processors than elements
519  {
520  // Throw an error
521  std::ostringstream error_stream;
522  error_stream << "You have tried to distribute a problem\n"
523  << "but there are less elements than processors.\n"
524  << "Please re-run with more elements!\n"
525  << "Please also ensure that actions_before_distribute().\n"
526  << "and actions_after_distribute() are correctly set up.\n"
527  << std::endl;
528  throw OomphLibError(
529  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
530  }
531  else
532  {
533  // We only distribute uniformly-refined meshes; buffer the case where
534  // either mesh is not uniformly refined
535  bool a_mesh_is_not_uniformly_refined = false;
536  unsigned n_mesh = nsub_mesh();
537  if (n_mesh == 0)
538  {
539  // Check refinement levels
540  if (TreeBasedRefineableMeshBase* mmesh_pt =
541  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(0)))
542  {
543  unsigned min_ref_level = 0;
544  unsigned max_ref_level = 0;
545  mmesh_pt->get_refinement_levels(min_ref_level, max_ref_level);
546  // If they are not the same
547  if (max_ref_level != min_ref_level)
548  {
549  a_mesh_is_not_uniformly_refined = true;
550  }
551  }
552  }
553  else
554  {
555  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
556  {
557  // Check refinement levels for each mesh individually
558  // (one mesh is allowed to be "more uniformly refined" than another)
559  if (TreeBasedRefineableMeshBase* mmesh_pt =
560  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(i_mesh)))
561  {
562  unsigned min_ref_level = 0;
563  unsigned max_ref_level = 0;
564  mmesh_pt->get_refinement_levels(min_ref_level, max_ref_level);
565  // If they are not the same
566  if (max_ref_level != min_ref_level)
567  {
568  a_mesh_is_not_uniformly_refined = true;
569  }
570  }
571  }
572  }
573 
574  // If any mesh is not uniformly refined
575  if (a_mesh_is_not_uniformly_refined)
576  {
577  // Again it may make more sense to throw an error here as the user
578  // will probably not be running a problem that is small enough to
579  // fit the whole of on each processor
580  std::ostringstream error_stream;
581  error_stream << "You have tried to distribute a problem\n"
582  << "but at least one of your meshes is no longer\n"
583  << "uniformly refined. In order to preserve the Tree\n"
584  << "and TreeForest structure, Problem::distribute() can\n"
585  << "only be called while meshes are uniformly refined.\n"
586  << std::endl;
587  throw OomphLibError(
588  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
589  }
590  else
591  {
592  // Is there any global data? If so, distributing the problem won't work
593  if (nglobal_data() > 0)
594  {
595  std::ostringstream error_stream;
596  error_stream << "You have tried to distribute a problem\n"
597  << "and there is some global data.\n"
598  << "This is not likely to work...\n"
599  << std::endl;
600  throw OomphLibError(error_stream.str(),
601  OOMPH_CURRENT_FUNCTION,
602  OOMPH_EXCEPTION_LOCATION);
603  }
604 
605  double t_start = 0;
607  {
608  t_start = TimingHelpers::timer();
609  }
610 
611 
612 #ifdef PARANOID
613  unsigned old_ndof = ndof();
614 #endif
615 
616  // Need to partition the global mesh before distributing
617  Mesh* global_mesh_pt = mesh_pt();
618 
619  // Vector listing the affiliation of each element
620  unsigned nelem = global_mesh_pt->nelement();
621  Vector<unsigned> element_domain(nelem);
622 
623  // Number of elements that I'm in charge of, based on any
624  // incoming partitioning
625  unsigned n_my_elements = 0;
626 
627  // Have we used the pre-set partitioning
628  bool used_preset_partitioning = false;
629 
630  // Partition the mesh, unless the partition has already been passed in
631  // If it hasn't then the sum of all the entries of the vector should be
632  // 0
633  unsigned sum_element_partition = 0;
634  unsigned n_part = element_partition.size();
635  for (unsigned e = 0; e < n_part; e++)
636  {
637  // ... another one for me.
638  if (int(element_partition[e]) == my_rank) n_my_elements++;
639 
640  sum_element_partition += element_partition[e];
641  }
642  if (sum_element_partition == 0)
643  {
644  oomph_info << "INFO: using METIS to partition elements" << std::endl;
645  partition_global_mesh(global_mesh_pt, doc_info, element_domain);
646  used_preset_partitioning = false;
647  }
648  else
649  {
650  oomph_info << "INFO: using pre-set partition of elements"
651  << std::endl;
652  used_preset_partitioning = true;
653  element_domain = element_partition;
654  }
655 
656  // Set the GLOBAL Mesh as being distributed
657  global_mesh_pt->set_communicator_pt(this->communicator_pt());
658 
659  double t_end = 0.0;
661  {
662  t_end = TimingHelpers::timer();
663  oomph_info << "Time for partitioning of global mesh: "
664  << t_end - t_start << std::endl;
665  t_start = TimingHelpers::timer();
666  }
667 
668  // Store how many elements we had in the various sub-meshes
669  // before actions_before_distribute() (which may empty some of
670  // them).
671  Vector<unsigned> n_element_in_old_submesh(n_mesh);
672  if (n_mesh != 0)
673  {
674  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
675  {
676  unsigned nsub_elem = mesh_pt(i_mesh)->nelement();
677  n_element_in_old_submesh[i_mesh] = nsub_elem;
678  }
679  }
680 
681  // Partitioning complete; call actions before distribute
683 
685  {
686  t_end = TimingHelpers::timer();
687  oomph_info << "Time for actions before distribute: "
688  << t_end - t_start << std::endl;
689  }
690 
691  // This next bit is cheap -- omit timing
692  // t_start = TimingHelpers::timer();
693 
694  // Number of submeshes (NB: some may have been deleted in
695  // actions_after_distribute())
696  n_mesh = nsub_mesh();
697 
698 
699  // Prepare vector of vectors for submesh element domains
700  Vector<Vector<unsigned>> submesh_element_domain(n_mesh);
701 
702  // The submeshes need to know their own element domains.
703  // Also if any meshes have been emptied we ignore their
704  // partitioning in the vector that we return from here
705  return_element_domain.reserve(element_domain.size());
706  if (n_mesh != 0)
707  {
708  unsigned count = 0;
709  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
710  {
711  unsigned nsub_elem = mesh_pt(i_mesh)->nelement();
712  submesh_element_domain[i_mesh].resize(nsub_elem);
713  unsigned nsub_elem_old = n_element_in_old_submesh[i_mesh];
714  for (unsigned e = 0; e < nsub_elem_old; e++)
715  {
716  if (nsub_elem_old == nsub_elem)
717  {
718  submesh_element_domain[i_mesh][e] = element_domain[count];
719  return_element_domain.push_back(element_domain[count]);
720  }
721  // return_element_domain.push_back(element_domain[count]);
722  count++;
723  }
724  }
725  }
726  else
727  {
728  return_element_domain = element_domain;
729  }
730 
732  {
733  t_start = TimingHelpers::timer();
734  }
735 
736  // Setup the map between "root" element and number in global mesh
737  // (currently used in the load_balance() routines)
738 
739  // This map is only established for structured meshes, then we
740  // need to check here the type of mesh
741  if (n_mesh == 0)
742  {
743  // Check if the only one mesh is an structured mesh
744  bool structured_mesh = true;
745  TriangleMeshBase* tri_mesh_pt =
746  dynamic_cast<TriangleMeshBase*>(mesh_pt(0));
747  if (tri_mesh_pt != 0)
748  {
749  structured_mesh = false;
750  } // if (tri_mesh_pt != 0)
751  if (structured_mesh)
752  {
753  const unsigned n_ele = global_mesh_pt->nelement();
754  Base_mesh_element_pt.resize(n_ele);
756  for (unsigned e = 0; e < n_ele; e++)
757  {
758  GeneralisedElement* el_pt = global_mesh_pt->element_pt(e);
760  Base_mesh_element_pt[e] = el_pt;
761  } // for (e<n_ele)
762  } // A TreeBaseMesh mesh
763  } // if (n_mesh==0)
764  else
765  {
766  // If we have submeshes then we only add those elements that
767  // belong to structured meshes, but first compute the number
768  // of total elements in the structured meshes
769  unsigned nglobal_element = 0;
770  // Store which submeshes are structured
771  std::vector<bool> is_structured_mesh(n_mesh);
772  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
773  {
774  TriangleMeshBase* tri_mesh_pt =
775  dynamic_cast<TriangleMeshBase*>(mesh_pt(i_mesh));
776  if (tri_mesh_pt != 0)
777  {
778  // Set the flags to indicate this is not an structured
779  // mesh
780  is_structured_mesh[i_mesh] = false;
781  } // if (tri_mesh_pt != 0)
782  else
783  {
784  // Set the flags to indicate this is an structured
785  // mesh
786  is_structured_mesh[i_mesh] = true;
787  } // else if (tri_mesh_pt != 0)
788  // Check if mesh is an structured mesh
789  if (is_structured_mesh[i_mesh])
790  {
791  nglobal_element += mesh_pt(i_mesh)->nelement();
792  } // A TreeBaseMesh mesh
793  } // for (i_mesh<n_mesh)
794 
795  // Once computed the number of elements, then resize the
796  // structure
797  Base_mesh_element_pt.resize(nglobal_element);
799  unsigned counter = 0;
800  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
801  {
802  // Check if mesh is an structured mesh
803  if (is_structured_mesh[i_mesh])
804  {
805  const unsigned n_ele = mesh_pt(i_mesh)->nelement();
806  for (unsigned e = 0; e < n_ele; e++)
807  {
808  GeneralisedElement* el_pt = mesh_pt(i_mesh)->element_pt(e);
809  Base_mesh_element_number_plus_one[el_pt] = counter + 1;
810  Base_mesh_element_pt[counter] = el_pt;
811  // Inrease the global element number
812  counter++;
813  } // for (e<n_ele)
814  } // An structured mesh
815  } // for (i_mesh<n_mesh)
816 
817 #ifdef PARANOID
818  if (counter != nglobal_element)
819  {
820  std::ostringstream error_stream;
821  error_stream
822  << "The number of global elements (" << nglobal_element
823  << ") is not the sameas the number of\nadded elements ("
824  << counter << ") to the Base_mesh_element_pt data "
825  << "structure!!!\n\n";
826  throw OomphLibError(error_stream.str(),
827  "Problem::distribute()",
828  OOMPH_EXCEPTION_LOCATION);
829  } // if (counter != nglobal_element)
830 #endif // #ifdef PARANOID
831 
832  } // else if (n_mesh==0)
833 
834  // Wipe everything if a pre-determined partitioning
835  // didn't specify ANY elements for this processor
836  // (typically happens during restarts with larger number
837  // of processors -- in this case we really want an empty
838  // processor rather than one with any "kept" halo elements)
839  bool overrule_keep_as_halo_element_status = false;
840  if ((n_my_elements == 0) && (used_preset_partitioning))
841  {
842  oomph_info << "INFO: We're over-ruling the \"keep as halo element\"\n"
843  << " status because the preset partitioning\n"
844  << " didn't place ANY elements on this processor,\n"
845  << " probably because of a restart on a larger \n"
846  << " number of processors\n";
847  overrule_keep_as_halo_element_status = true;
848  }
849 
850 
851  // Distribute the (sub)meshes (i.e. sort out their halo lookup schemes)
852  Vector<GeneralisedElement*> deleted_element_pt;
853  if (n_mesh == 0)
854  {
855  global_mesh_pt->distribute(this->communicator_pt(),
856  element_domain,
857  deleted_element_pt,
858  doc_info,
859  report_stats,
860  overrule_keep_as_halo_element_status);
861  }
862  else // There are submeshes, "distribute" each one separately
863  {
864  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
865  {
866  if (report_stats)
867  {
868  oomph_info << "Distributing submesh " << i_mesh << std::endl
869  << "--------------------" << std::endl;
870  }
871  // Set the doc_info number to reflect the submesh
872  doc_info.number() = i_mesh;
873  mesh_pt(i_mesh)->distribute(this->communicator_pt(),
874  submesh_element_domain[i_mesh],
875  deleted_element_pt,
876  doc_info,
877  report_stats,
878  overrule_keep_as_halo_element_status);
879  }
880  // Rebuild the global mesh
882  }
883 
884  // Null out information associated with deleted elements
885  unsigned n_del = deleted_element_pt.size();
886  for (unsigned e = 0; e < n_del; e++)
887  {
888  GeneralisedElement* el_pt = deleted_element_pt[e];
889  unsigned old_el_number = Base_mesh_element_number_plus_one[el_pt] - 1;
891  Base_mesh_element_pt[old_el_number] = 0;
892  }
893 
895  {
896  t_end = TimingHelpers::timer();
897  oomph_info << "Time for mesh-level distribution: " << t_end - t_start
898  << std::endl;
899  t_start = TimingHelpers::timer();
900  }
901 
902  // Now the problem has been distributed
904 
905  // Call actions after distribute
907 
909  {
910  t_end = TimingHelpers::timer();
911  oomph_info << "Time for actions after distribute: " << t_end - t_start
912  << std::endl;
913  t_start = TimingHelpers::timer();
914  }
915 
916  // Re-assign the equation numbers (incl synchronisation if reqd)
917  unsigned n_dof = assign_eqn_numbers();
918  oomph_info << "Number of equations: " << n_dof << std::endl;
919 
921  {
922  t_end = TimingHelpers::timer();
923  oomph_info << "Time for re-assigning eqn numbers (in distribute): "
924  << t_end - t_start << std::endl;
925  }
926 
927 
928 #ifdef PARANOID
929  if (n_dof != old_ndof)
930  {
931  std::ostringstream error_stream;
932  error_stream
933  << "Number of dofs in distribute() has changed "
934  << "from " << old_ndof << " to " << n_dof << "\n"
935  << "Check that you've implemented any necessary "
936  "actions_before/after\n"
937  << "distribute functions, e.g. to pin redundant pressure dofs"
938  << " etc.\n";
939  throw OomphLibError(error_stream.str(),
940  OOMPH_CURRENT_FUNCTION,
941  OOMPH_EXCEPTION_LOCATION);
942  }
943 #endif
944 
945  } // end if to check for uniformly refined mesh(es)
946 
947  } // end if to check number of processors vs. number of elements etc.
948 
949 
950  // Force re-analysis of time spent on assembly each
951  // elemental Jacobian
953  Elemental_assembly_time.clear();
954 
955  // Return the partition vector used in the distribution
956  return return_element_domain;
957  }
958 
959  //==================================================================
960  /// Partition the global mesh, return vector specifying the processor
961  /// number for each element. Virtual so that it can be overloaded by
962  /// any user; the default is to use METIS to perform the partitioning
963  /// (with a bit of cleaning up afterwards to sort out "special cases").
964  //==================================================================
965  void Problem::partition_global_mesh(Mesh*& global_mesh_pt,
966  DocInfo& doc_info,
967  Vector<unsigned>& element_domain,
968  const bool& report_stats)
969  {
970  // Storage for number of processors and current processor
971  int n_proc = this->communicator_pt()->nproc();
972  int rank = this->communicator_pt()->my_rank();
973 
974  std::ostringstream filename;
975  std::ofstream some_file;
976 
977  // Doc the original mesh on proc 0
978  //--------------------------------
979  if (doc_info.is_doc_enabled())
980  {
981  if (rank == 0)
982  {
983  filename << doc_info.directory() << "/complete_mesh"
984  << doc_info.number() << ".dat";
985  global_mesh_pt->output(filename.str().c_str(), 5);
986  }
987  }
988 
989  // Partition the mesh
990  //-------------------
991  // METIS Objective (0: minimise edge cut; 1: minimise total comm volume)
992  unsigned objective = 0;
993 
994  // Do the partitioning
995  unsigned nelem = 0;
996  if (this->communicator_pt()->my_rank() == 0)
997  {
998  METIS::partition_mesh(this, n_proc, objective, element_domain);
999  nelem = element_domain.size();
1000  }
1001  MPI_Bcast(&nelem, 1, MPI_UNSIGNED, 0, this->communicator_pt()->mpi_comm());
1002  element_domain.resize(nelem);
1003  MPI_Bcast(&element_domain[0],
1004  nelem,
1005  MPI_UNSIGNED,
1006  0,
1007  this->communicator_pt()->mpi_comm());
1008 
1009  // On very coarse meshes with larger numbers of processors, METIS
1010  // occasionally returns an element_domain Vector for which a particular
1011  // processor has no elements affiliated to it; the following fixes this
1012 
1013  // Convert element_domain to integer storage
1014  Vector<int> int_element_domain(nelem);
1015  for (unsigned e = 0; e < nelem; e++)
1016  {
1017  int_element_domain[e] = element_domain[e];
1018  }
1019 
1020  // Global storage for number of elements on each process
1021  int my_number_of_elements = 0;
1022  Vector<int> number_of_elements(n_proc, 0);
1023 
1024  for (unsigned e = 0; e < nelem; e++)
1025  {
1026  if (int_element_domain[e] == rank)
1027  {
1028  my_number_of_elements++;
1029  }
1030  }
1031 
1032  // Communicate the correct value for each single process into
1033  // the global storage vector
1034  MPI_Allgather(&my_number_of_elements,
1035  1,
1036  MPI_INT,
1037  &number_of_elements[0],
1038  1,
1039  MPI_INT,
1040  this->communicator_pt()->mpi_comm());
1041 
1042  // If a process has no elements then switch an element with the
1043  // process with the largest number of elements, assuming
1044  // that it still has enough elements left to share
1045  int max_number_of_elements = 0;
1046  int process_with_max_elements = 0;
1047  for (int d = 0; d < n_proc; d++)
1048  {
1049  if (number_of_elements[d] == 0)
1050  {
1051  // Find the process with maximum number of elements
1052  if (max_number_of_elements <= 1)
1053  {
1054  for (int dd = 0; dd < n_proc; dd++)
1055  {
1056  if (number_of_elements[dd] > max_number_of_elements)
1057  {
1058  max_number_of_elements = number_of_elements[dd];
1059  process_with_max_elements = dd;
1060  }
1061  }
1062  }
1063 
1064  // Check that this number of elements is okay for sharing
1065  if (max_number_of_elements <= 1)
1066  {
1067  // Throw an error if elements can't be shared
1068  std::ostringstream error_stream;
1069  error_stream << "No process has more than 1 element, and\n"
1070  << "at least one process has no elements!\n"
1071  << "Suggest rerunning with more refinement.\n"
1072  << std::endl;
1073  throw OomphLibError(error_stream.str(),
1074  OOMPH_CURRENT_FUNCTION,
1075  OOMPH_EXCEPTION_LOCATION);
1076  }
1077 
1078  // Loop over the element domain vector and switch
1079  // one value for process "process_with_max_elements" with d
1080  for (unsigned e = 0; e < nelem; e++)
1081  {
1082  if (int_element_domain[e] == process_with_max_elements)
1083  {
1084  int_element_domain[e] = d;
1085  // Change the numbers associated with these processes
1086  number_of_elements[d]++;
1087  number_of_elements[process_with_max_elements]--;
1088  // Reduce the number of elements available on "max" process
1089  max_number_of_elements--;
1090  // Inform the user that a switch has taken place
1091  if (report_stats)
1092  {
1093  oomph_info << "INFO: Switched element domain at position " << e
1094  << std::endl
1095  << "from process " << process_with_max_elements
1096  << " to process " << d << std::endl
1097  << "which was given no elements by METIS partition"
1098  << std::endl;
1099  }
1100  // Only need to do this once for this element loop, otherwise
1101  // this will take all the elements from "max" process and put them
1102  // in process d, thus leaving essentially the same problem!
1103  break;
1104  }
1105  }
1106  }
1107  }
1108 
1109  // Reassign new values to the element_domain vector
1110  for (unsigned e = 0; e < nelem; e++)
1111  {
1112  element_domain[e] = int_element_domain[e];
1113  }
1114 
1115  unsigned count_elements = 0;
1116  for (unsigned e = 0; e < nelem; e++)
1117  {
1118  if (int(element_domain[e]) == rank)
1119  {
1120  count_elements++;
1121  }
1122  }
1123 
1124  if (report_stats)
1125  {
1126  oomph_info << "I have " << count_elements
1127  << " elements from this partition" << std::endl
1128  << std::endl;
1129  }
1130  }
1131 
1132  //==================================================================
1133  /// (Irreversibly) prune halo(ed) elements and nodes, usually
1134  /// after another round of refinement, to get rid of
1135  /// excessively wide halo layers. Note that the current
1136  /// mesh will be now regarded as the base mesh and no unrefinement
1137  /// relative to it will be possible once this function
1138  /// has been called.
1139  //==================================================================
1141  const bool& report_stats)
1142  {
1143  // Storage for number of processors and current processor
1144  int n_proc = this->communicator_pt()->nproc();
1145 
1146  // Has the problem been distributed yet?
1148  {
1149  oomph_info
1150  << "WARNING: Problem::prune_halo_elements_and_nodes() was called on a "
1151  << "non-distributed Problem!" << std::endl;
1152  oomph_info << "Ignoring your request..." << std::endl;
1153  }
1154  else
1155  {
1156  // There are no halo layers to prune if it's a single-process job
1157  if (n_proc == 1)
1158  {
1159  oomph_info
1160  << "WARNING: You've tried to prune halo layers on a problem\n"
1161  << "with only one processor: this is unnecessary.\n"
1162  << "Ignoring your request." << std::endl
1163  << std::endl;
1164  }
1165  else
1166  {
1167 #ifdef PARANOID
1168  unsigned old_ndof = ndof();
1169 #endif
1170 
1171  double t_start = 0.0;
1173  {
1174  t_start = TimingHelpers::timer();
1175  }
1176 
1177  // Call actions before distribute
1179 
1180  double t_end = 0.0;
1182  {
1183  t_end = TimingHelpers::timer();
1184  oomph_info << "Time for actions_before_distribute() in "
1185  << "Problem::prune_halo_elements_and_nodes(): "
1186  << t_end - t_start << std::endl;
1187  t_start = TimingHelpers::timer();
1188  }
1189 
1190  // Associate all elements with root in current Base mesh
1191  unsigned nel = Base_mesh_element_pt.size();
1192  std::map<GeneralisedElement*, unsigned>
1193  old_base_element_number_plus_one;
1194  std::vector<bool> old_root_is_halo_or_non_existent(nel, true);
1195  for (unsigned e = 0; e < nel; e++)
1196  {
1197  // Get the base element
1198  GeneralisedElement* base_el_pt = Base_mesh_element_pt[e];
1199 
1200  // Does it exist locally?
1201  if (base_el_pt != 0)
1202  {
1203  // Check if it's a halo element
1204  if (!base_el_pt->is_halo())
1205  {
1206  old_root_is_halo_or_non_existent[e] = false;
1207  }
1208 
1209  // Not refineable: It's only the element iself
1210  RefineableElement* ref_el_pt = 0;
1211  ref_el_pt = dynamic_cast<RefineableElement*>(base_el_pt);
1212  if (ref_el_pt == 0)
1213  {
1214  old_base_element_number_plus_one[base_el_pt] = e + 1;
1215  }
1216  // Refineable: Get entire tree of elements
1217  else
1218  {
1219  Vector<Tree*> tree_pt;
1220  ref_el_pt->tree_pt()->stick_all_tree_nodes_into_vector(tree_pt);
1221  unsigned ntree = tree_pt.size();
1222  for (unsigned t = 0; t < ntree; t++)
1223  {
1224  old_base_element_number_plus_one[tree_pt[t]->object_pt()] =
1225  e + 1;
1226  }
1227  }
1228  }
1229  }
1230 
1231 
1233  {
1234  t_end = TimingHelpers::timer();
1235  oomph_info << "Time for setup old root elements in "
1236  << "Problem::prune_halo_elements_and_nodes(): "
1237  << t_end - t_start << std::endl;
1238  t_start = TimingHelpers::timer();
1239  }
1240 
1241 
1242  // Now remember the old number of base elements
1243  unsigned nel_base_old = nel;
1244 
1245 
1246  // Prune the halo elements and nodes of the mesh(es)
1247  Vector<GeneralisedElement*> deleted_element_pt;
1248  unsigned n_mesh = nsub_mesh();
1249  if (n_mesh == 0)
1250  {
1251  // Prune halo elements and nodes for the (single) global mesh
1253  deleted_element_pt, doc_info, report_stats);
1254  }
1255  else
1256  {
1257  // Loop over individual submeshes and prune separately
1258  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
1259  {
1261  deleted_element_pt, doc_info, report_stats);
1262  }
1263 
1264  // Rebuild the global mesh
1266  }
1267 
1269  {
1270  t_end = TimingHelpers::timer();
1271  oomph_info << "Total time for all mesh-level prunes in "
1272  << "Problem::prune_halo_elements_and_nodes(): "
1273  << t_end - t_start << std::endl;
1274  t_start = TimingHelpers::timer();
1275  }
1276 
1277  // Loop over all elements in newly rebuilt mesh (which contains
1278  // all element in "tree order"), find the roots
1279  // (which are either non-refineable elements or refineable elements
1280  // whose tree representations are TreeRoots)
1281  std::map<FiniteElement*, bool> root_el_done;
1282 
1283  // Vector storing vectors of pointers to new base elements associated
1284  // with the same old base element
1286  new_base_element_associated_with_old_base_element(nel_base_old);
1287 
1288  unsigned n_meshes = n_mesh;
1289  // Change the value for the number of submeshes if there is only
1290  // one mesh so that the loop below works if we have only one
1291  // mesh
1292  if (n_meshes == 0)
1293  {
1294  n_meshes = 1;
1295  }
1296 
1297  // Store which submeshes, if there are some are structured
1298  // meshes
1299  std::vector<bool> is_structured_mesh(n_meshes);
1300 
1301  // Loop over all elements in the rebuilt mesh, but make sure
1302  // that we are only looping over the structured meshes
1303  nel = 0;
1304  for (unsigned i_mesh = 0; i_mesh < n_meshes; i_mesh++)
1305  {
1306  TriangleMeshBase* tri_mesh_pt =
1307  dynamic_cast<TriangleMeshBase*>(mesh_pt(i_mesh));
1308  if (!(tri_mesh_pt != 0))
1309  {
1310  // Mark the mesh as structured mesh
1311  is_structured_mesh[i_mesh] = true;
1312  // Add the number of elements
1313  nel += mesh_pt(i_mesh)->nelement();
1314  } // if (!(tri_mesh_pt!=0))
1315  else
1316  {
1317  // Mark the mesh as nonstructured mesh
1318  is_structured_mesh[i_mesh] = false;
1319  } // else if (!(tri_mesh_pt!=0))
1320  } // for (i_mesh < n_mesh)
1321 
1322  // Go for all the meshes (if there are submeshes)
1323  for (unsigned i_mesh = 0; i_mesh < n_meshes; i_mesh++)
1324  {
1325  // Only work with the elements in the mesh if it is an
1326  // structured mesh
1327  if (is_structured_mesh[i_mesh])
1328  {
1329  // Get the number of elements in the submesh
1330  const unsigned nele_submesh = mesh_pt(i_mesh)->nelement();
1331  for (unsigned e = 0; e < nele_submesh; e++)
1332  {
1333  // Get the element
1334  GeneralisedElement* el_pt = mesh_pt(i_mesh)->element_pt(e);
1335 
1336  // Not refineable: It's definitely a new base element
1337  RefineableElement* ref_el_pt = 0;
1338  ref_el_pt = dynamic_cast<RefineableElement*>(el_pt);
1339  if (ref_el_pt == 0)
1340  {
1341  unsigned old_base_el_no =
1342  old_base_element_number_plus_one[el_pt] - 1;
1343  new_base_element_associated_with_old_base_element
1344  [old_base_el_no]
1345  .push_back(el_pt);
1346  }
1347  // Refineable
1348  else
1349  {
1350  // Is it a tree root (after pruning)? In that case it's
1351  // a new base element
1352  if (dynamic_cast<TreeRoot*>(ref_el_pt->tree_pt()))
1353  {
1354  unsigned old_base_el_no =
1355  old_base_element_number_plus_one[el_pt] - 1;
1356  new_base_element_associated_with_old_base_element
1357  [old_base_el_no]
1358  .push_back(el_pt);
1359  }
1360  else
1361  {
1362  // Get associated root element
1363  FiniteElement* root_el_pt =
1364  ref_el_pt->tree_pt()->root_pt()->object_pt();
1365 
1366  if (!root_el_done[root_el_pt])
1367  {
1368  root_el_done[root_el_pt] = true;
1369  unsigned old_base_el_no =
1370  old_base_element_number_plus_one[el_pt] - 1;
1371  new_base_element_associated_with_old_base_element
1372  [old_base_el_no]
1373  .push_back(root_el_pt);
1374  }
1375  }
1376  }
1377  } // for (e < nele_submesh)
1378  } // if (is_structured_mesh[i_mesh])
1379  } // for (i_mesh < n_mesh)
1380 
1381  // Create a vector that stores how many new root/base elements
1382  // got spawned from each old root/base element in the global mesh
1383  Vector<unsigned> local_n_new_root(nel_base_old);
1384 #ifdef PARANOID
1385  Vector<unsigned> n_new_root_back(nel_base_old);
1386 #endif
1387  for (unsigned e = 0; e < nel_base_old; e++)
1388  {
1389  local_n_new_root[e] =
1390  new_base_element_associated_with_old_base_element[e].size();
1391 
1392 #ifdef PARANOID
1393  // Backup so we can check that halo data was consistent
1394  n_new_root_back[e] = local_n_new_root[e];
1395 #endif
1396  }
1397 
1399  {
1400  t_end = TimingHelpers::timer();
1401  oomph_info << "Time for setup of new base elements in "
1402  << "Problem::prune_halo_elements_and_nodes(): "
1403  << t_end - t_start << std::endl;
1404  t_start = TimingHelpers::timer();
1405  }
1406 
1407  // Now do reduce operation to get information for all
1408  // old root/base elements -- the pruned (halo!) base elements contain
1409  // fewer associated new roots.
1410  Vector<unsigned> n_new_root(nel_base_old);
1411  MPI_Allreduce(&local_n_new_root[0],
1412  &n_new_root[0],
1413  nel_base_old,
1414  MPI_UNSIGNED,
1415  MPI_MAX,
1416  this->communicator_pt()->mpi_comm());
1417 
1418 
1420  {
1421  t_end = TimingHelpers::timer();
1422  oomph_info << "Time for allreduce in "
1423  << "Problem::prune_halo_elements_and_nodes(): "
1424  << t_end - t_start << std::endl;
1425  t_start = TimingHelpers::timer();
1426  }
1427 
1428  // Find out total number of base elements
1429  unsigned nel_base_new = 0;
1430  for (unsigned e = 0; e < nel_base_old; e++)
1431  {
1432  // Increment
1433  nel_base_new += n_new_root[e];
1434 
1435 #ifdef PARANOID
1436  // If we already had data for this root previously then
1437  // the data ought to be consistent afterwards (since taking
1438  // the max of consistent numbers shouldn't change things -- this
1439  // deals with halo/haloed elements)
1440  if (!old_root_is_halo_or_non_existent[e])
1441  {
1442  if (n_new_root_back[e] != 0)
1443  {
1444  if (n_new_root_back[e] != n_new_root[e])
1445  {
1446  std::ostringstream error_stream;
1447  error_stream
1448  << "Number of new root elements spawned from old root " << e
1449  << ": " << n_new_root[e] << "\nis not consistent"
1450  << " with previous value: " << n_new_root_back[e]
1451  << std::endl;
1452  throw OomphLibError(error_stream.str(),
1453  OOMPH_CURRENT_FUNCTION,
1454  OOMPH_EXCEPTION_LOCATION);
1455  }
1456  }
1457  }
1458 
1459 #endif
1460  }
1461 
1462  // Reset base_mesh information
1463  Base_mesh_element_pt.clear();
1464  Base_mesh_element_pt.resize(nel_base_new, 0);
1466 
1467  // Now enumerate the new base/root elements consistently
1468  unsigned count = 0;
1469  for (unsigned e = 0; e < nel_base_old; e++)
1470  {
1471  // Old root is non-halo: Just add the new roots into the
1472  // new lookup scheme consecutively
1473  if (!old_root_is_halo_or_non_existent[e])
1474  {
1475  // Loop over new root/base element
1476  unsigned n_new_root =
1477  new_base_element_associated_with_old_base_element[e].size();
1478  for (unsigned j = 0; j < n_new_root; j++)
1479  {
1480  // Store new root/base element
1481  GeneralisedElement* el_pt =
1482  new_base_element_associated_with_old_base_element[e][j];
1483  Base_mesh_element_pt[count] = el_pt;
1484  Base_mesh_element_number_plus_one[el_pt] = count + 1;
1485 
1486  // Bump counter
1487  count++;
1488  }
1489  }
1490  // Old root element is halo so skip insertion (i.e. leave
1491  // entries in lookup schemes nulled) but increase counter to
1492  // ensure consistency between processors
1493  else
1494  {
1495  unsigned nskip = n_new_root[e];
1496  count += nskip;
1497  }
1498  }
1499 
1500  // Re-setup the map between "root" element and number in global mesh
1501  // (used in the load_balance() routines)
1503 
1504 
1506  {
1507  t_end = TimingHelpers::timer();
1508  oomph_info << "Time for finishing off base mesh info "
1509  << "Problem::prune_halo_elements_and_nodes(): "
1510  << t_end - t_start << std::endl;
1511  t_start = TimingHelpers::timer();
1512  }
1513 
1514 
1515  // Call actions after distribute
1517 
1518 
1520  {
1521  t_end = TimingHelpers::timer();
1522  oomph_info << "Time for actions_after_distribute() "
1523  << "Problem::prune_halo_elements_and_nodes(): "
1524  << t_end - t_start << std::endl;
1525  t_start = TimingHelpers::timer();
1526  }
1527 
1528 
1529  // Re-assign the equation numbers (incl synchronisation if reqd)
1530 #ifdef PARANOID
1531  unsigned n_dof = assign_eqn_numbers();
1532 #else
1534 #endif
1535 
1536 
1538  {
1539  t_end = TimingHelpers::timer();
1540  oomph_info << "Time for assign_eqn_numbers() "
1541  << "Problem::prune_halo_elements_and_nodes(): "
1542  << t_end - t_start << std::endl;
1543  t_start = TimingHelpers::timer();
1544  }
1545 
1546 
1547 #ifdef PARANOID
1549  {
1550  if (n_dof != old_ndof)
1551  {
1552  std::ostringstream error_stream;
1553  error_stream
1554  << "Number of dofs in prune_halo_elements_and_nodes() has "
1555  "changed "
1556  << "from " << old_ndof << " to " << n_dof << "\n"
1557  << "Check that you've implemented any necessary "
1558  "actions_before/after"
1559  << "\nadapt/distribute functions, e.g. to pin redundant pressure"
1560  << " dofs etc.\n";
1561  throw OomphLibError(error_stream.str(),
1562  OOMPH_CURRENT_FUNCTION,
1563  OOMPH_EXCEPTION_LOCATION);
1564  }
1565  }
1566 #endif
1567  }
1568  }
1569  }
1570 
1571 
1572 #endif
1573 
1574 
1575  //===================================================================
1576  /// Build a single (global) mesh from a number
1577  /// of submeshes which are passed as a vector of pointers to the
1578  /// submeshes. The ordering is not necessarily optimal.
1579  //==============================================================
1581  {
1582 #ifdef PARANOID
1583  // Has a global mesh already been built
1584  if (Mesh_pt != 0)
1585  {
1586  std::string error_message = "Problem::build_global_mesh() called,\n";
1587  error_message += " but a global mesh has already been built:\n";
1588  error_message += "Problem::Mesh_pt is not zero!\n";
1589 
1590  throw OomphLibError(
1591  error_message, OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
1592  }
1593  // Check that there are submeshes
1594  if (Sub_mesh_pt.size() == 0)
1595  {
1596  std::string error_message = "Problem::build_global_mesh() called,\n";
1597  error_message += " but there are no submeshes:\n";
1598  error_message += "Problem::Sub_mesh_pt has no entries\n";
1599 
1600  throw OomphLibError(
1601  error_message, OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
1602  }
1603 #endif
1604 
1605  // Create an empty mesh
1606  Mesh_pt = new Mesh();
1607 
1608  // Call the rebuild function to construct the mesh
1610  }
1611 
1612  //====================================================================
1613  /// If one of the submeshes has changed (e.g. by
1614  /// mesh adaptation) we need to update the global mesh.
1615  /// \b Note: The nodes boundary information refers to the
1616  /// boundary numbers within the submesh!
1617  /// N.B. This is essentially the same function as the Mesh constructor
1618  /// that assembles a single global mesh from submeshes
1619  //=====================================================================
1621  {
1622  // Use the function in mesh to merge the submeshes into this one
1624  }
1625 
1626 
1627  //================================================================
1628  /// Add a timestepper to the problem. The function will automatically
1629  /// create or resize the Time object so that it contains the appropriate
1630  /// number of levels of storage.
1631  //================================================================
1632  void Problem::add_time_stepper_pt(TimeStepper* const& time_stepper_pt)
1633  {
1634  // Add the timestepper to the vector
1635  Time_stepper_pt.push_back(time_stepper_pt);
1636 
1637  // Find the number of timesteps required by the timestepper
1638  unsigned ndt = time_stepper_pt->ndt();
1639 
1640  // If time has not been allocated, create time object with the
1641  // required number of time steps
1642  if (Time_pt == 0)
1643  {
1644  Time_pt = new Time(ndt);
1645  oomph_info << "Created Time with " << ndt << " timesteps" << std::endl;
1646  }
1647  else
1648  {
1649  // If the required number of time steps is greater than currently stored
1650  // resize the time storage
1651  if (ndt > Time_pt->ndt())
1652  {
1653  Time_pt->resize(ndt);
1654  oomph_info << "Resized Time to include " << ndt << " timesteps"
1655  << std::endl;
1656  }
1657  // Otherwise report that we are OK
1658  else
1659  {
1660  oomph_info << "Time object already has storage for " << ndt
1661  << " timesteps" << std::endl;
1662  }
1663  }
1664 
1665  // Pass the pointer to time to the timestepper
1667  }
1668 
1669  //================================================================
1670  /// Set the explicit time stepper for the problem and also
1671  /// ensure that a time object has been created.
1672  //================================================================
1674  ExplicitTimeStepper* const& explicit_time_stepper_pt)
1675  {
1676  // Set the explicit time stepper
1678 
1679  // If time has not been allocated, create time object with the
1680  // required number of time steps
1681  if (Time_pt == 0)
1682  {
1683  Time_pt = new Time(0);
1684  oomph_info << "Created Time with storage for no previous timestep"
1685  << std::endl;
1686  }
1687  else
1688  {
1689  oomph_info << "Time object already exists " << std::endl;
1690  }
1691  }
1692 
1693 
1694 #ifdef OOMPH_HAS_MPI
1695 
1696  //================================================================
1697  /// Set default first and last elements for parallel assembly
1698  /// of non-distributed problem.
1699  //================================================================
1701  {
1703  {
1704  // Minimum number of elements per processor if there are fewer elements
1705  // than processors
1706  unsigned min_el = 10;
1707 
1708  // Resize and make default assignments
1709  int n_proc = this->communicator_pt()->nproc();
1710  unsigned n_elements = Mesh_pt->nelement();
1711  First_el_for_assembly.resize(n_proc, 0);
1712  Last_el_plus_one_for_assembly.resize(n_proc, 0);
1713 
1714  // In the absence of any better knowledge distribute work evenly
1715  // over elements
1716  unsigned range = 0;
1717  unsigned lo_proc = 0;
1718  unsigned hi_proc = n_proc - 1;
1719  if (int(n_elements) >= n_proc)
1720  {
1721  range = unsigned(double(n_elements) / double(n_proc));
1722  }
1723  else
1724  {
1725  range = min_el;
1726  lo_proc = 0;
1727  hi_proc = unsigned(double(n_elements) / double(min_el));
1728  }
1729 
1730  for (int p = lo_proc; p <= int(hi_proc); p++)
1731  {
1732  First_el_for_assembly[p] = p * range;
1733 
1734  unsigned last_el_plus_one = (p + 1) * range;
1735  if (last_el_plus_one > n_elements) last_el_plus_one = n_elements;
1736  Last_el_plus_one_for_assembly[p] = last_el_plus_one;
1737  }
1738 
1739  // Last one needs to incorporate any dangling elements
1740  if (int(n_elements) >= n_proc)
1741  {
1742  Last_el_plus_one_for_assembly[n_proc - 1] = n_elements;
1743  }
1744 
1745  // Doc
1746  if (n_proc > 1)
1747  {
1749  {
1750  oomph_info << "Problem is not distributed. Parallel assembly of "
1751  << "Jacobian uses default partitioning: " << std::endl;
1752  for (int p = 0; p < n_proc; p++)
1753  {
1754  if (Last_el_plus_one_for_assembly[p] != 0)
1755  {
1756  oomph_info << "Proc " << p << " assembles from element "
1757  << First_el_for_assembly[p] << " to "
1758  << Last_el_plus_one_for_assembly[p] - 1 << " \n";
1759  }
1760  else
1761  {
1762  oomph_info << "Proc " << p << " assembles no elements\n";
1763  }
1764  }
1765  }
1766  }
1767  }
1768  }
1769 
1770 
1771  //=======================================================================
1772  /// Helper function to re-assign the first and last elements to be
1773  /// assembled by each processor during parallel assembly for
1774  /// non-distributed problem.
1775  //=======================================================================
1777  {
1778  // Wait until all processes have completed/timed their assembly
1779  MPI_Barrier(this->communicator_pt()->mpi_comm());
1780 
1781  // Storage for number of processors and current processor
1782  int n_proc = this->communicator_pt()->nproc();
1783  int rank = this->communicator_pt()->my_rank();
1784 
1785  // Don't bother to update if we've got fewer elements than
1786  // processors
1787  unsigned nel = Elemental_assembly_time.size();
1788  if (int(nel) < n_proc)
1789  {
1790  oomph_info << "Not re-computing distribution of elemental assembly\n"
1791  << "because there are fewer elements than processors\n";
1792  return;
1793  }
1794 
1795  // Setup vectors storing the number of element timings to be sent
1796  // and the offset in the final vector
1797  Vector<int> receive_count(n_proc);
1798  Vector<int> displacement(n_proc);
1799  int offset = 0;
1800  for (int p = 0; p < n_proc; p++)
1801  {
1802  // Default distribution of labour
1803  unsigned el_lo = First_el_for_assembly[p];
1804  unsigned el_hi = Last_el_plus_one_for_assembly[p] - 1;
1805 
1806  // Number of timings to be sent and offset from start in
1807  // final vector
1808  receive_count[p] = el_hi - el_lo + 1;
1809  displacement[p] = offset;
1810  offset += el_hi - el_lo + 1;
1811  }
1812 
1813  // Make temporary c-style array to avoid over-writing in Gatherv below
1814  double* el_ass_time = new double[nel];
1815  for (unsigned e = 0; e < nel; e++)
1816  {
1817  el_ass_time[e] = Elemental_assembly_time[e];
1818  }
1819 
1820  // Gather timings on root processor
1821  unsigned nel_local =
1823  MPI_Gatherv(&el_ass_time[First_el_for_assembly[rank]],
1824  nel_local,
1825  MPI_DOUBLE,
1827  &receive_count[0],
1828  &displacement[0],
1829  MPI_DOUBLE,
1830  0,
1831  this->communicator_pt()->mpi_comm());
1832  delete[] el_ass_time;
1833 
1834  // Vector of first and last elements for each processor
1835  Vector<Vector<int>> first_and_last_element(n_proc);
1836  for (int p = 0; p < n_proc; p++)
1837  {
1838  first_and_last_element[p].resize(2);
1839  }
1840 
1841  // Re-distribute work
1842  if (rank == 0)
1843  {
1845  {
1846  oomph_info
1847  << std::endl
1848  << "Re-assigning distribution of element assembly over processors:"
1849  << std::endl;
1850  }
1851 
1852  // Get total assembly time
1853  double total = 0.0;
1854  unsigned n_elements = Mesh_pt->nelement();
1855  for (unsigned e = 0; e < n_elements; e++)
1856  {
1857  total += Elemental_assembly_time[e];
1858  }
1859 
1860  // Target load per processor
1861  double target_load = total / double(n_proc);
1862 
1863  // We're on the root processor: Always start with the first element
1864  int proc = 0;
1865  first_and_last_element[0][0] = 0;
1866 
1867  // Highest element we can help ourselves to if we want to leave
1868  // at least one element for all subsequent processors
1869  unsigned max_el_avail = n_elements - n_proc;
1870 
1871  // Initialise total work allocated
1872  total = 0.0;
1873  for (unsigned e = 0; e < n_elements; e++)
1874  {
1875  total += Elemental_assembly_time[e];
1876 
1877  // Once we have reached the target load or we've used up practically
1878  // all the elements...
1879  if ((total > target_load) || (e == max_el_avail))
1880  {
1881  // Last element for current processor
1882  first_and_last_element[proc][1] = e;
1883 
1884  // Provided that we are not on the last processor
1885  if (proc < (n_proc - 1))
1886  {
1887  // Set first element for next one
1888  first_and_last_element[proc + 1][0] = e + 1;
1889 
1890  // Move on to the next processor
1891  proc++;
1892  }
1893 
1894  // Can have one more...
1895  max_el_avail++;
1896 
1897  // Re-initialise the time
1898  total = 0.0;
1899  } // end of test for "total exceeds target"
1900  }
1901 
1902 
1903  // Last element for last processor
1904  first_and_last_element[n_proc - 1][1] = n_elements - 1;
1905 
1906 
1907  // The following block should probably be paranoidified away
1908  // but we've screwed the logic up so many times that I feel
1909  // it's safer to keep it...
1910  bool wrong = false;
1911  std::ostringstream error_stream;
1912  for (int p = 0; p < n_proc - 1; p++)
1913  {
1914  unsigned first_of_current = first_and_last_element[p][0];
1915  unsigned last_of_current = first_and_last_element[p][1];
1916  if (first_of_current > last_of_current)
1917  {
1918  wrong = true;
1919  error_stream << "Error: First/last element of proc " << p << ": "
1920  << first_of_current << " " << last_of_current
1921  << std::endl;
1922  }
1923  unsigned first_of_next = first_and_last_element[p + 1][0];
1924  if (first_of_next != (last_of_current + 1))
1925  {
1926  wrong = true;
1927  error_stream << "Error: First element of proc " << p + 1 << ": "
1928  << first_of_next << " and last element of proc " << p
1929  << ": " << last_of_current << std::endl;
1930  }
1931  }
1932  if (wrong)
1933  {
1934  throw OomphLibError(
1935  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
1936  }
1937 
1938 
1939  // THIS TIDY UP SHOULD NO LONGER BE REQUIRED AND CAN GO AT SOME POINT
1940 
1941  // //If we haven't got to the end of the processor list then
1942  // //need to shift things about slightly because the processors
1943  // //at the end will be empty.
1944  // //This can occur when you have very fast assembly times and the
1945  // //rounding errors mean that the targets are achieved before all
1946  // processors
1947  // //have been visited.
1948  // //Happens a lot when you massively oversubscribe the CPUs (which was
1949  // //only ever for testing!)
1950  // if (proc!=n_proc-1)
1951  // {
1952  // oomph_info
1953  // << "First pass did not allocate elements on every processor\n";
1954  // oomph_info <<
1955  // "Moving elements so that each processor has at least one\n";
1956 
1957  // //Work out number of empty processos
1958  // unsigned n_empty_processors = n_proc - proc + 1;
1959 
1960  // //Loop over the processors that do have elements
1961  // //and work out how many we need to steal elements from
1962  // unsigned n_element_on_processors=0;
1963  // do
1964  // {
1965  // //Step down the processors
1966  // --proc;
1967  // //Add the current processor to the number of empty processors
1968  // //because the elements have to be shared between processors
1969  // //including the one(s) on which they are currently stored.
1970  // ++n_empty_processors;
1971  // n_element_on_processors +=
1972  // (first_and_last_element[proc][1] -
1973  // first_and_last_element[proc][0] + 1);
1974  // }
1975  // while(n_element_on_processors < n_empty_processors);
1976 
1977  // //Should now be able to put one element on each processor
1978  // //Start from the end and do so
1979  // unsigned current_element = n_elements-1;
1980  // for(int p=n_proc-1;p>proc;p--)
1981  // {
1982  // first_and_last_element[p][1] = current_element;
1983  // first_and_last_element[p][0] = --current_element;
1984  // }
1985 
1986  // //Now for the last processor we touched, just adjust the final value
1987  // first_and_last_element[proc][1] = current_element;
1988  // }
1989  // //Otherwise just put the rest of the elements on the final
1990  // //processor
1991  // else
1992  // {
1993  // // Last one
1994  // first_and_last_element[n_proc-1][1]=n_elements-1;
1995  // }
1996 
1997 
1998  // END PRESUMED-TO-BE-UNNECESSARY BLOCK...
1999 
2000 
2001  // Now communicate the information
2002 
2003  // Set local informationt for this (root) processor
2004  First_el_for_assembly[0] = first_and_last_element[0][0];
2005  Last_el_plus_one_for_assembly[0] = first_and_last_element[0][1] + 1;
2006 
2008  {
2009  oomph_info << "Processor " << 0 << " assembles Jacobians"
2010  << " from elements " << first_and_last_element[0][0]
2011  << " to " << first_and_last_element[0][1] << " "
2012  << std::endl;
2013  }
2014 
2015  // Only now can we send the information to the other processors
2016  for (int p = 1; p < n_proc; ++p)
2017  {
2018  MPI_Send(&first_and_last_element[p][0],
2019  2,
2020  MPI_INT,
2021  p,
2022  0,
2023  this->communicator_pt()->mpi_comm());
2024 
2025 
2027  {
2028  oomph_info << "Processor " << p << " assembles Jacobians"
2029  << " from elements " << first_and_last_element[p][0]
2030  << " to " << first_and_last_element[p][1] << " "
2031  << std::endl;
2032  }
2033  }
2034  }
2035  // Receive first and last element from root on non-master processors
2036  else
2037  {
2038  Vector<int> aux(2);
2039  MPI_Status status;
2040  MPI_Recv(&aux[0],
2041  2,
2042  MPI_INT,
2043  0,
2044  0,
2045  this->communicator_pt()->mpi_comm(),
2046  &status);
2047  First_el_for_assembly[rank] = aux[0];
2048  Last_el_plus_one_for_assembly[rank] = aux[1] + 1;
2049  }
2050 
2051  // Wipe all others
2052  for (int p = 0; p < n_proc; p++)
2053  {
2054  if (p != rank)
2055  {
2056  First_el_for_assembly[p] = 0;
2058  }
2059  }
2060 
2061  // The equations assembled by this processor may have changed so
2062  // we must resize the sparse assemble with arrays previous allocation
2064  }
2065 
2066 #endif
2067 
2068  //================================================================
2069  /// Assign all equation numbers for problem: Deals with global
2070  /// data (= data that isn't attached to any elements) and then
2071  /// does the equation numbering for the elements. Bool argument
2072  /// can be set to false to ignore assigning local equation numbers
2073  /// (necessary in the parallel implementation of locate_zeta
2074  /// between multiple meshes).
2075  //================================================================
2077  const bool& assign_local_eqn_numbers)
2078  {
2079  // Check that the global mesh has been build
2080 #ifdef PARANOID
2081  if (Mesh_pt == 0)
2082  {
2083  std::ostringstream error_stream;
2084  error_stream << "Global mesh does not exist, so equation numbers cannot "
2085  "be assigned.\n";
2086  // Check for sub meshes
2087  if (nsub_mesh() == 0)
2088  {
2089  error_stream << "There aren't even any sub-meshes in the Problem.\n"
2090  << "You can set the global mesh directly by using\n"
2091  << "Problem::mesh_pt() = my_mesh_pt;\n"
2092  << "OR you can use Problem::add_sub_mesh(mesh_pt); "
2093  << "to add a sub mesh.\n";
2094  }
2095  else
2096  {
2097  error_stream << "There are " << nsub_mesh() << " sub-meshes.\n";
2098  }
2099  error_stream << "You need to call Problem::build_global_mesh() to create "
2100  "a global mesh\n"
2101  << "from the sub-meshes.\n\n";
2102 
2103  throw OomphLibError(
2104  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
2105  }
2106 #endif
2107 
2108  // Number of submeshes
2109  unsigned n_sub_mesh = Sub_mesh_pt.size();
2110 
2111 #ifdef OOMPH_HAS_MPI
2112 
2113  // Storage for number of processors
2114  int n_proc = this->communicator_pt()->nproc();
2115 
2116 
2117  if (n_proc > 1)
2118  {
2119  // Force re-analysis of time spent on assembly each
2120  // elemental Jacobian
2122  Elemental_assembly_time.clear();
2123  }
2124  else
2125  {
2127  }
2128 
2129  // Re-distribution of elements over processors during assembly
2130  // must be recomputed
2132  {
2133  // Set default first and last elements for parallel assembly
2134  // of non-distributed problem.
2136  }
2137 
2138 #endif
2139 
2140 
2141  double t_start = 0.0;
2143  {
2144  t_start = TimingHelpers::timer();
2145  }
2146 
2147  // Loop over all elements in the mesh and set up any additional
2148  // dependencies that they may have (e.g. storing the geometric
2149  // Data, i.e. Data that affects an element's shape in elements
2150  // with algebraic node-update functions
2151  unsigned nel = Mesh_pt->nelement();
2152  for (unsigned e = 0; e < nel; e++)
2153  {
2155  }
2156 
2157 #ifdef OOMPH_HAS_MPI
2158  // Complete setup of dependencies for external halo elements too
2159  unsigned n_mesh = this->nsub_mesh();
2160  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
2161  {
2162  for (int iproc = 0; iproc < n_proc; iproc++)
2163  {
2164  unsigned n_ext_halo_el = mesh_pt(i_mesh)->nexternal_halo_element(iproc);
2165  for (unsigned e = 0; e < n_ext_halo_el; e++)
2166  {
2167  mesh_pt(i_mesh)
2168  ->external_halo_element_pt(iproc, e)
2170  }
2171  }
2172  }
2173 #endif
2174 
2175 
2176  double t_end = 0.0;
2178  {
2179  t_end = TimingHelpers::timer();
2180  oomph_info
2181  << "Time for complete setup of dependencies in assign_eqn_numbers: "
2182  << t_end - t_start << std::endl;
2183  }
2184 
2185 
2186  // Initialise number of dofs for reserve below
2187  unsigned n_dof = 0;
2188 
2189  // Potentially loop over remainder of routine, possible re-visiting all
2190  // those parts that must be redone, following the removal of duplicate
2191  // external halo data.
2192  for (unsigned loop_count = 0; loop_count < 2; loop_count++)
2193  {
2194  //(Re)-set the dof pointer to zero length because entries are
2195  // pushed back onto it -- if it's not reset here then we get into
2196  // trouble during mesh refinement when we reassign all dofs
2197  Dof_pt.resize(0);
2198 
2199  // Reserve from previous allocation if we're going around again
2200  Dof_pt.reserve(n_dof);
2201 
2202  // Reset the equation number
2203  unsigned long equation_number = 0;
2204 
2205  // Now set equation numbers for the global Data
2206  unsigned Nglobal_data = nglobal_data();
2207  for (unsigned i = 0; i < Nglobal_data; i++)
2208  {
2209  Global_data_pt[i]->assign_eqn_numbers(equation_number, Dof_pt);
2210  }
2211 
2213  {
2214  t_start = TimingHelpers::timer();
2215  }
2216 
2217  // Call assign equation numbers on the global mesh
2219 
2220  // Deal with the spine meshes additional numbering
2221  // If there is only one mesh
2222  if (n_sub_mesh == 0)
2223  {
2224  if (SpineMesh* const spine_mesh_pt = dynamic_cast<SpineMesh*>(Mesh_pt))
2225  {
2226  n_dof = spine_mesh_pt->assign_global_spine_eqn_numbers(Dof_pt);
2227  }
2228  }
2229  // Otherwise loop over the sub meshes
2230  else
2231  {
2232  // Assign global equation numbers first
2233  for (unsigned i = 0; i < n_sub_mesh; i++)
2234  {
2235  if (SpineMesh* const spine_mesh_pt =
2236  dynamic_cast<SpineMesh*>(Sub_mesh_pt[i]))
2237  {
2238  n_dof = spine_mesh_pt->assign_global_spine_eqn_numbers(Dof_pt);
2239  }
2240  }
2241  }
2242 
2244  {
2245  t_end = TimingHelpers::timer();
2246  oomph_info
2247  << "Time for assign_global_eqn_numbers in assign_eqn_numbers: "
2248  << t_end - t_start << std::endl;
2249  t_start = TimingHelpers::timer();
2250  }
2251 
2252 
2253 #ifdef OOMPH_HAS_MPI
2254 
2255  // reset previous allocation
2257 
2258  // Only synchronise if the problem has actually been
2259  // distributed.
2261  {
2262  // Synchronise the equation numbers and return the total
2263  // number of degrees of freedom in the overall problem
2264  // Do not assign local equation numbers -- we're doing this
2265  // below.
2266  n_dof = synchronise_eqn_numbers(false);
2267  }
2268  // ..else just setup the Dof_distribution_pt
2269  // NOTE: this is setup by synchronise_eqn_numbers(...)
2270  // if Problem_has_been_distributed
2271  else
2272 #endif
2273  {
2274  Dof_distribution_pt->build(Communicator_pt, n_dof, false);
2275  }
2276 
2278  {
2279  t_end = TimingHelpers::timer();
2280  oomph_info << "Time for Problem::synchronise_eqn_numbers in "
2281  << "Problem::assign_eqn_numbers: " << t_end - t_start
2282  << std::endl;
2283  }
2284 
2285 
2286 #ifdef OOMPH_HAS_MPI
2287 
2288 
2289  // Now remove duplicate data in external halo elements
2291  {
2293  {
2294  t_start = TimingHelpers::timer();
2295  }
2296 
2297  // Monitor if we've actually changed anything
2298  bool actually_removed_some_data = false;
2299 
2300  // Only do it once!
2301  if (loop_count == 0)
2302  {
2303  if (n_sub_mesh == 0)
2304  {
2305  remove_duplicate_data(Mesh_pt, actually_removed_some_data);
2306  }
2307  else
2308  {
2309  for (unsigned i = 0; i < n_sub_mesh; i++)
2310  {
2311  bool tmp_actually_removed_some_data = false;
2313  tmp_actually_removed_some_data);
2314  if (tmp_actually_removed_some_data)
2315  actually_removed_some_data = true;
2316  }
2317  }
2318  }
2319 
2320 
2322  {
2323  t_end = TimingHelpers::timer();
2324  std::stringstream tmp;
2325  tmp << "Time for calls to Problem::remove_duplicate_data in "
2326  << "Problem::assign_eqn_numbers: " << t_end - t_start
2327  << " ; have ";
2328  if (!actually_removed_some_data)
2329  {
2330  tmp << " not ";
2331  }
2332  tmp << " removed some/any data.\n";
2333  oomph_info << tmp.str();
2334  t_start = TimingHelpers::timer();
2335  }
2336 
2337  // Break out of the loop if we haven't done anything here.
2338  unsigned status = 0;
2339  if (actually_removed_some_data) status = 1;
2340 
2341  // Allreduce to check if anyone has removed any data
2342  unsigned overall_status = 0;
2343  MPI_Allreduce(&status,
2344  &overall_status,
2345  1,
2346  MPI_UNSIGNED,
2347  MPI_MAX,
2348  this->communicator_pt()->mpi_comm());
2349 
2350 
2352  {
2353  t_end = TimingHelpers::timer();
2354  std::stringstream tmp;
2355  tmp
2356  << "Time for MPI_Allreduce after Problem::remove_duplicate_data in "
2357  << "Problem::assign_eqn_numbers: " << t_end - t_start << std::endl;
2358  oomph_info << tmp.str();
2359  t_start = TimingHelpers::timer();
2360  }
2361 
2362  // Bail out if we haven't done anything here
2363  if (overall_status != 1)
2364  {
2365  break;
2366  }
2367 
2368  // Big tidy up: Remove null pointers from halo/haloed node storage
2369  // for all meshes (this involves comms and therefore must be
2370  // performed outside loop over meshes so the all-to-all is only
2371  // done once)
2373 
2374  // Time it...
2376  {
2377  double t_end = TimingHelpers::timer();
2378  oomph_info << "Total time for "
2379  << "Problem::remove_null_pointers_from_external_halo_node_"
2380  "storage(): "
2381  << t_end - t_start << std::endl;
2382  }
2383  }
2384  else
2385  {
2386  // Problem not distributed; no need for another loop
2387  break;
2388  }
2389 
2390 #else
2391 
2392  // Serial run: Again no need for a second loop
2393  break;
2394 
2395 #endif
2396 
2397  } // end of loop over fcts that need to be re-executed if
2398  // we've removed duplicate external data
2399 
2400 
2401  // Resize the sparse assemble with arrays previous allocation
2403 
2404 
2406  {
2407  t_start = TimingHelpers::timer();
2408  }
2409 
2410  // Finally assign local equations
2411  if (assign_local_eqn_numbers)
2412  {
2413  if (n_sub_mesh == 0)
2414  {
2416  }
2417  else
2418  {
2419  for (unsigned i = 0; i < n_sub_mesh; i++)
2420  {
2421  Sub_mesh_pt[i]->assign_local_eqn_numbers(
2423  }
2424  }
2425  }
2426 
2428  {
2429  t_end = TimingHelpers::timer();
2430  oomph_info << "Total time for all Mesh::assign_local_eqn_numbers in "
2431  << "Problem::assign_eqn_numbers: " << t_end - t_start
2432  << std::endl;
2433  }
2434 
2435 
2436  // and return the total number of DOFs
2437  return n_dof;
2438  }
2439  //================================================================
2440  /// Function to describe the dofs in terms of the global
2441  /// equation number, i.e. what type of value (nodal value of
2442  /// a Node; value in a Data object; value of internal Data in an
2443  /// element; etc) is the unknown with a certain global equation number.
2444  /// Output stream defaults to oomph_info.
2445  //================================================================
2446  void Problem::describe_dofs(std::ostream& out) const
2447  {
2448  // Check that the global mesh has been build
2449 #ifdef PARANOID
2450  if (Mesh_pt == 0)
2451  {
2452  std::ostringstream error_stream;
2453  error_stream
2454  << "Global mesh does not exist, so equation numbers cannot be found.\n";
2455  // Check for sub meshes
2456  if (nsub_mesh() == 0)
2457  {
2458  error_stream << "There aren't even any sub-meshes in the Problem.\n"
2459  << "You can set the global mesh directly by using\n"
2460  << "Problem::mesh_pt() = my_mesh_pt;\n"
2461  << "OR you can use Problem::add_sub_mesh(mesh_pt); "
2462  << "to add a sub mesh.\n";
2463  }
2464  else
2465  {
2466  error_stream << "There are " << nsub_mesh() << " sub-meshes.\n";
2467  }
2468  error_stream << "You need to call Problem::build_global_mesh() to create "
2469  "a global mesh\n"
2470  << "from the sub-meshes.\n\n";
2471 
2472  throw OomphLibError(
2473  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
2474  }
2475 #endif
2476 
2477  out
2478  << "Although this program will describe the degrees of freedom in the \n"
2479  << "problem, it will do so using the typedef for the elements. This is \n"
2480  << "not neccesarily human readable, but there is a solution.\n"
2481  << "Pipe your program's output through c++filt, with the argument -t.\n"
2482  << "e.g. \"./two_d_multi_poisson | c++filt -t > ReadableOutput.txt\".\n "
2483  << "(Disregarding the quotes)\n\n\n";
2484 
2485  out << "Classifying Global Equation Numbers" << std::endl;
2486  out << std::string(80, '-') << std::endl;
2487 
2488  // Number of submeshes
2489  unsigned n_sub_mesh = Sub_mesh_pt.size();
2490 
2491  // Classify Global dofs
2492  unsigned Nglobal_data = nglobal_data();
2493  for (unsigned i = 0; i < Nglobal_data; i++)
2494  {
2495  std::stringstream conversion;
2496  conversion << " in Global Data " << i << ".";
2497  std::string in(conversion.str());
2498  Global_data_pt[i]->describe_dofs(out, in);
2499  }
2500 
2501  // Put string in limiting scope.
2502  {
2503  // Descend into assignment for mesh.
2504  std::string in(" in Problem's Only Mesh.");
2505  Mesh_pt->describe_dofs(out, in);
2506  }
2507 
2508  // Deal with the spine meshes additional numbering:
2509  // If there is only one mesh:
2510  if (n_sub_mesh == 0)
2511  {
2512  if (SpineMesh* const spine_mesh_pt = dynamic_cast<SpineMesh*>(Mesh_pt))
2513  {
2514  std::string in(" in Problem's Only SpineMesh.");
2515  spine_mesh_pt->describe_spine_dofs(out, in);
2516  }
2517  }
2518  // Otherwise loop over the sub meshes
2519  else
2520  {
2521  // Assign global equation numbers first
2522  for (unsigned i = 0; i < n_sub_mesh; i++)
2523  {
2524  if (SpineMesh* const spine_mesh_pt =
2525  dynamic_cast<SpineMesh*>(Sub_mesh_pt[i]))
2526  {
2527  std::stringstream conversion;
2528  conversion << " in Sub-SpineMesh " << i << ".";
2529  std::string in(conversion.str());
2530  spine_mesh_pt->describe_spine_dofs(out, in);
2531  } // end if.
2532  } // end for.
2533  } // end else.
2534 
2535 
2536  out << std::string(80, '\\') << std::endl;
2537  out << std::string(80, '\\') << std::endl;
2538  out << std::string(80, '\\') << std::endl;
2539  out << "Classifying global eqn numbers in terms of elements." << std::endl;
2540  out << std::string(80, '-') << std::endl;
2541  out << "Eqns | Source" << std::endl;
2542  out << std::string(80, '-') << std::endl;
2543 
2544  if (n_sub_mesh == 0)
2545  {
2546  std::string in(" in Problem's Only Mesh.");
2547  Mesh_pt->describe_local_dofs(out, in);
2548  }
2549  else
2550  {
2551  for (unsigned i = 0; i < n_sub_mesh; i++)
2552  {
2553  std::stringstream conversion;
2554  conversion << " in Sub-Mesh " << i << ".";
2555  std::string in(conversion.str());
2556  Sub_mesh_pt[i]->describe_local_dofs(out, in);
2557  } // End for
2558  } // End else
2559  } // End problem::describe_dofs(...)
2560 
2561 
2562  //================================================================
2563  /// Get the vector of dofs, i.e. a vector containing the current
2564  /// values of all unknowns.
2565  //================================================================
2567  {
2568  // Find number of dofs
2569  const unsigned long n_dof = ndof();
2570 
2571  // Resize the vector
2572  dofs.build(Dof_distribution_pt, 0.0);
2573 
2574  // Copy dofs into vector
2575  for (unsigned long l = 0; l < n_dof; l++)
2576  {
2577  dofs[l] = *Dof_pt[l];
2578  }
2579  }
2580 
2581  /// Get history values of dofs in a double vector.
2582  void Problem::get_dofs(const unsigned& t, DoubleVector& dofs) const
2583  {
2584 #ifdef PARANOID
2585  if (distributed())
2586  {
2587  throw OomphLibError("Not designed for distributed problems",
2588  OOMPH_EXCEPTION_LOCATION,
2589  OOMPH_CURRENT_FUNCTION);
2590  // might work, not sure
2591  }
2592 #endif
2593 
2594  // Resize the vector
2595  dofs.build(Dof_distribution_pt, 0.0);
2596 
2597  // First deal with global data
2598  unsigned Nglobal_data = nglobal_data();
2599  for (unsigned i = 0; i < Nglobal_data; i++)
2600  {
2601  for (unsigned j = 0, nj = Global_data_pt[i]->nvalue(); j < nj; j++)
2602  {
2603  // For each data get the equation number and copy out the value.
2604  int eqn_number = Global_data_pt[i]->eqn_number(j);
2605  if (eqn_number >= 0)
2606  {
2607  dofs[eqn_number] = Global_data_pt[i]->value(t, j);
2608  }
2609  }
2610  }
2611 
2612  // Next element internal data
2613  for (unsigned i = 0, ni = mesh_pt()->nelement(); i < ni; i++)
2614  {
2615  GeneralisedElement* ele_pt = mesh_pt()->element_pt(i);
2616  for (unsigned j = 0, nj = ele_pt->ninternal_data(); j < nj; j++)
2617  {
2618  Data* d_pt = ele_pt->internal_data_pt(j);
2619  for (unsigned k = 0, nk = d_pt->nvalue(); k < nk; k++)
2620  {
2621  int eqn_number = d_pt->eqn_number(k);
2622  if (eqn_number >= 0)
2623  {
2624  dofs[eqn_number] = d_pt->value(t, k);
2625  }
2626  }
2627  }
2628  }
2629 
2630  // Now the nodes
2631  for (unsigned i = 0, ni = mesh_pt()->nnode(); i < ni; i++)
2632  {
2633  Node* node_pt = mesh_pt()->node_pt(i);
2634  for (unsigned j = 0, nj = node_pt->nvalue(); j < nj; j++)
2635  {
2636  // For each node get the equation number and copy out the value.
2637  int eqn_number = node_pt->eqn_number(j);
2638  if (eqn_number >= 0)
2639  {
2640  dofs[eqn_number] = node_pt->value(t, j);
2641  }
2642  }
2643  }
2644  }
2645 
2646 
2647 #ifdef OOMPH_HAS_MPI
2648 
2649  //=======================================================================
2650  /// Private helper function to remove repeated data
2651  /// in external haloed elements associated with specified mesh.
2652  /// Bool is true if some data was removed -- this usually requires
2653  /// re-running through certain parts of the equation numbering procedure.
2654  //======================================================================
2655  void Problem::remove_duplicate_data(Mesh* const& mesh_pt,
2656  bool& actually_removed_some_data)
2657  {
2658  // // // Taken out again by MH -- clutters up output
2659  // // Doc timings if required
2660  // double t_start=0.0;
2661  // if (Global_timings::Doc_comprehensive_timings)
2662  // {
2663  // t_start=TimingHelpers::timer();
2664  // }
2665 
2666  int n_proc = this->communicator_pt()->nproc();
2667  int my_rank = this->communicator_pt()->my_rank();
2668 
2669  // Initialise
2670  actually_removed_some_data = false;
2671 
2672  // Each individual container of external halo nodes has unique
2673  // nodes/equation numbers, but there may be some duplication between
2674  // two or more different containers; the following code checks for this
2675  // and removes the duplication by overwriting any data point with an already
2676  // existing eqn number with the original data point which had the eqn no.
2677 
2678  // // Storage for existing nodes, enumerated by first non-negative
2679  // // global equation number
2680  // unsigned n_dof=ndof();
2681 
2682  // Note: This used to be
2683  // Vector<Node*> global_node_pt(n_dof,0);
2684  // but this is a total killer! Memory allocation is extremely
2685  // costly and only relatively few entries are used so use
2686  // map:
2687  std::map<unsigned, Node*> global_node_pt;
2688 
2689  // Only do each retained node once
2690  std::map<Node*, bool> node_done;
2691 
2692  // Loop over existing "normal" elements in mesh
2693  unsigned n_element = mesh_pt->nelement();
2694  for (unsigned e = 0; e < n_element; e++)
2695  {
2696  FiniteElement* el_pt =
2697  dynamic_cast<FiniteElement*>(mesh_pt->element_pt(e));
2698  if (el_pt != 0)
2699  {
2700  // Loop over nodes
2701  unsigned n_node = el_pt->nnode();
2702  for (unsigned j = 0; j < n_node; j++)
2703  {
2704  Node* nod_pt = el_pt->node_pt(j);
2705 
2706  // Have we already done the node?
2707  if (!node_done[nod_pt])
2708  {
2709  node_done[nod_pt] = true;
2710 
2711  // Loop over values stored at node (if any) to find
2712  // the first non-negative eqn number
2713  unsigned first_non_negative_eqn_number_plus_one = 0;
2714  unsigned n_val = nod_pt->nvalue();
2715  for (unsigned i_val = 0; i_val < n_val; i_val++)
2716  {
2717  int eqn_no = nod_pt->eqn_number(i_val);
2718  if (eqn_no >= 0)
2719  {
2720  first_non_negative_eqn_number_plus_one = eqn_no + 1;
2721  break;
2722  }
2723  }
2724 
2725  // If we haven't found a non-negative eqn number check
2726  // eqn numbers associated with solid data (if any)
2727  if (first_non_negative_eqn_number_plus_one == 0)
2728  {
2729  // Is it a solid node?
2730  SolidNode* solid_nod_pt = dynamic_cast<SolidNode*>(nod_pt);
2731  if (solid_nod_pt != 0)
2732  {
2733  // Loop over values stored at node (if any) to find
2734  // the first non-negative eqn number
2735  unsigned n_val = solid_nod_pt->variable_position_pt()->nvalue();
2736  for (unsigned i_val = 0; i_val < n_val; i_val++)
2737  {
2738  int eqn_no =
2739  solid_nod_pt->variable_position_pt()->eqn_number(i_val);
2740  if (eqn_no >= 0)
2741  {
2742  first_non_negative_eqn_number_plus_one = eqn_no + 1;
2743  break;
2744  }
2745  }
2746  }
2747  }
2748 
2749  // Associate node with first non negative global eqn number
2750  if (first_non_negative_eqn_number_plus_one > 0)
2751  {
2752  global_node_pt[first_non_negative_eqn_number_plus_one - 1] =
2753  nod_pt;
2754  }
2755 
2756 
2757  // Take into account master nodes too
2758  if (dynamic_cast<RefineableElement*>(el_pt) != 0)
2759  {
2760  int n_cont_int_values = dynamic_cast<RefineableElement*>(el_pt)
2761  ->ncont_interpolated_values();
2762  for (int i_cont = -1; i_cont < n_cont_int_values; i_cont++)
2763  {
2764  if (nod_pt->is_hanging(i_cont))
2765  {
2766  HangInfo* hang_pt = nod_pt->hanging_pt(i_cont);
2767  unsigned n_master = hang_pt->nmaster();
2768  for (unsigned m = 0; m < n_master; m++)
2769  {
2770  Node* master_nod_pt = hang_pt->master_node_pt(m);
2771  if (!node_done[master_nod_pt])
2772  {
2773  node_done[master_nod_pt] = true;
2774 
2775  // Loop over values stored at node (if any) to find
2776  // the first non-negative eqn number
2777  unsigned first_non_negative_eqn_number_plus_one = 0;
2778  unsigned n_val = master_nod_pt->nvalue();
2779  for (unsigned i_val = 0; i_val < n_val; i_val++)
2780  {
2781  int eqn_no = master_nod_pt->eqn_number(i_val);
2782  if (eqn_no >= 0)
2783  {
2784  first_non_negative_eqn_number_plus_one = eqn_no + 1;
2785  break;
2786  }
2787  }
2788 
2789  // If we haven't found a non-negative eqn number check
2790  // eqn numbers associated with solid data (if any)
2791  if (first_non_negative_eqn_number_plus_one == 0)
2792  {
2793  // If this master is a SolidNode then add its extra
2794  // eqn numbers
2795  SolidNode* master_solid_nod_pt =
2796  dynamic_cast<SolidNode*>(master_nod_pt);
2797  if (master_solid_nod_pt != 0)
2798  {
2799  // Loop over values stored at node (if any) to find
2800  // the first non-negative eqn number
2801  unsigned n_val =
2802  master_solid_nod_pt->variable_position_pt()
2803  ->nvalue();
2804  for (unsigned i_val = 0; i_val < n_val; i_val++)
2805  {
2806  int eqn_no =
2807  master_solid_nod_pt->variable_position_pt()
2808  ->eqn_number(i_val);
2809  if (eqn_no >= 0)
2810  {
2811  first_non_negative_eqn_number_plus_one =
2812  eqn_no + 1;
2813  break;
2814  }
2815  }
2816  }
2817  }
2818  // Associate node with first non negative global
2819  // eqn number
2820  if (first_non_negative_eqn_number_plus_one > 0)
2821  {
2822  global_node_pt[first_non_negative_eqn_number_plus_one -
2823  1] = master_nod_pt;
2824  }
2825 
2826  } // End of not-yet-done hang node
2827  }
2828  }
2829  }
2830  }
2831  } // endif for node already done
2832  } // End of loop over nodes
2833  } // End of FiniteElement
2834 
2835  // Internal data equation numbers do not need to be added since
2836  // internal data cannot be shared between distinct elements, so
2837  // internal data on locally-stored elements can never be halo.
2838  }
2839 
2840  // Set to record duplicate nodes scheduled to be killed
2841  std::set<Node*> killed_nodes;
2842 
2843  // Now loop over the other processors from highest to lowest
2844  // (i.e. if there is a duplicate between these containers
2845  // then this will use the node on the highest numbered processor)
2846  for (int iproc = n_proc - 1; iproc >= 0; iproc--)
2847  {
2848  // Don't have external halo elements with yourself!
2849  if (iproc != my_rank)
2850  {
2851  // Loop over external halo elements with iproc
2852  // to remove the duplicates
2853  unsigned n_element = mesh_pt->nexternal_halo_element(iproc);
2854  for (unsigned e_ext = 0; e_ext < n_element; e_ext++)
2855  {
2856  FiniteElement* finite_ext_el_pt = dynamic_cast<FiniteElement*>(
2857  mesh_pt->external_halo_element_pt(iproc, e_ext));
2858  if (finite_ext_el_pt != 0)
2859  {
2860  // Loop over nodes
2861  unsigned n_node = finite_ext_el_pt->nnode();
2862  for (unsigned j = 0; j < n_node; j++)
2863  {
2864  Node* nod_pt = finite_ext_el_pt->node_pt(j);
2865 
2866  // Loop over values stored at node (if any) to find
2867  // the first non-negative eqn number
2868  unsigned first_non_negative_eqn_number_plus_one = 0;
2869  unsigned n_val = nod_pt->nvalue();
2870  for (unsigned i_val = 0; i_val < n_val; i_val++)
2871  {
2872  int eqn_no = nod_pt->eqn_number(i_val);
2873  if (eqn_no >= 0)
2874  {
2875  first_non_negative_eqn_number_plus_one = eqn_no + 1;
2876  break;
2877  }
2878  }
2879 
2880  // If we haven't found a non-negative eqn number check
2881  // eqn numbers associated with solid data (if any)
2882  if (first_non_negative_eqn_number_plus_one == 0)
2883  {
2884  // Is it a solid node?
2885  SolidNode* solid_nod_pt = dynamic_cast<SolidNode*>(nod_pt);
2886  if (solid_nod_pt != 0)
2887  {
2888  // Loop over values stored at node (if any) to find
2889  // the first non-negative eqn number
2890  unsigned n_val =
2891  solid_nod_pt->variable_position_pt()->nvalue();
2892  for (unsigned i_val = 0; i_val < n_val; i_val++)
2893  {
2894  int eqn_no =
2895  solid_nod_pt->variable_position_pt()->eqn_number(i_val);
2896  if (eqn_no >= 0)
2897  {
2898  first_non_negative_eqn_number_plus_one = eqn_no + 1;
2899  break;
2900  }
2901  }
2902  }
2903  }
2904 
2905  // Identified which node we're dealing with via first non-negative
2906  // global eqn number (if there is none, everything is pinned
2907  // and we don't give a damn...)
2908  if (first_non_negative_eqn_number_plus_one > 0)
2909  {
2910  Node* existing_node_pt =
2911  global_node_pt[first_non_negative_eqn_number_plus_one - 1];
2912 
2913  // Does this node already exist?
2914  if (existing_node_pt != 0)
2915  {
2916  // Record that we're about to cull one
2917  actually_removed_some_data = true;
2918 
2919  // It's a duplicate, so store the duplicated one for
2920  // later killing...
2921  Node* duplicated_node_pt = nod_pt;
2922  if (!node_done[duplicated_node_pt])
2923  {
2924  // Remove node from all boundaries
2925  std::set<unsigned>* boundaries_pt;
2926  duplicated_node_pt->get_boundaries_pt(boundaries_pt);
2927  if (boundaries_pt != 0)
2928  {
2929  Vector<unsigned> bound;
2930  unsigned nb = (*boundaries_pt).size();
2931  bound.reserve(nb);
2932  for (std::set<unsigned>::iterator it =
2933  (*boundaries_pt).begin();
2934  it != (*boundaries_pt).end();
2935  it++)
2936  {
2937  bound.push_back((*it));
2938  }
2939  for (unsigned i = 0; i < nb; i++)
2940  {
2941  mesh_pt->remove_boundary_node(bound[i],
2942  duplicated_node_pt);
2943  }
2944  }
2945 
2946  // Get ready to kill it
2947  killed_nodes.insert(duplicated_node_pt);
2948  unsigned i_proc = unsigned(iproc);
2950  duplicated_node_pt);
2951  }
2952 
2953 
2954  // Note: For now we're leaving the "dangling" (no longer
2955  // accessed masters where they are; they get cleaned
2956  // up next time we delete all the external storage
2957  // for the meshes so it's a temporary "leak" only...
2958  // At some point we should probably delete them properly too
2959 
2960 #ifdef PARANOID
2961 
2962  // Check that hang status of exiting and replacement node
2963  // matches
2964  if (dynamic_cast<RefineableElement*>(finite_ext_el_pt) != 0)
2965  {
2966  int n_cont_inter_values =
2967  dynamic_cast<RefineableElement*>(finite_ext_el_pt)
2968  ->ncont_interpolated_values();
2969  for (int i_cont = -1; i_cont < n_cont_inter_values;
2970  i_cont++)
2971  {
2972  unsigned n_master_orig = 0;
2973  if (finite_ext_el_pt->node_pt(j)->is_hanging(i_cont))
2974  {
2975  n_master_orig = finite_ext_el_pt->node_pt(j)
2976  ->hanging_pt(i_cont)
2977  ->nmaster();
2978 
2979  // Temporary leak: Resolve like this:
2980  // loop over all external halo nodes and identify the
2981  // the ones that are still reached by any of the
2982  // external elements. Kill the dangling ones.
2983  }
2984  unsigned n_master_replace = 0;
2985  if (existing_node_pt->is_hanging(i_cont))
2986  {
2987  n_master_replace =
2988  existing_node_pt->hanging_pt(i_cont)->nmaster();
2989  }
2990 
2991  if (n_master_orig != n_master_replace)
2992  {
2993  std::ostringstream error_stream;
2994  error_stream
2995  << "Number of master nodes for node to be replaced, "
2996  << n_master_orig << ", doesn't match"
2997  << "those of replacement node, " << n_master_replace
2998  << " for i_cont=" << i_cont << std::endl;
2999  {
3000  error_stream
3001  << "Nodal coordinates of replacement node:";
3002  unsigned ndim = existing_node_pt->ndim();
3003  for (unsigned i = 0; i < ndim; i++)
3004  {
3005  error_stream << existing_node_pt->x(i) << " ";
3006  }
3007  error_stream << "\n";
3008  error_stream << "The coordinates of its "
3009  << n_master_replace
3010  << " master nodes are: \n";
3011  for (unsigned k = 0; k < n_master_replace; k++)
3012  {
3013  Node* master_nod_pt =
3014  existing_node_pt->hanging_pt(i_cont)
3015  ->master_node_pt(k);
3016  unsigned ndim = master_nod_pt->ndim();
3017  for (unsigned i = 0; i < ndim; i++)
3018  {
3019  error_stream << master_nod_pt->x(i) << " ";
3020  }
3021  error_stream << "\n";
3022  }
3023  }
3024 
3025  {
3026  error_stream
3027  << "Nodal coordinates of node to be replaced:";
3028  unsigned ndim = finite_ext_el_pt->node_pt(j)->ndim();
3029  for (unsigned i = 0; i < ndim; i++)
3030  {
3031  error_stream << finite_ext_el_pt->node_pt(j)->x(i)
3032  << " ";
3033  }
3034  error_stream << "\n";
3035  error_stream << "The coordinates of its "
3036  << n_master_orig
3037  << " master nodes are: \n";
3038  for (unsigned k = 0; k < n_master_orig; k++)
3039  {
3040  Node* master_nod_pt = finite_ext_el_pt->node_pt(j)
3041  ->hanging_pt(i_cont)
3042  ->master_node_pt(k);
3043  unsigned ndim = master_nod_pt->ndim();
3044  for (unsigned i = 0; i < ndim; i++)
3045  {
3046  error_stream << master_nod_pt->x(i) << " ";
3047  }
3048  error_stream << "\n";
3049  }
3050  }
3051 
3052 
3053  throw OomphLibError(error_stream.str(),
3054  OOMPH_CURRENT_FUNCTION,
3055  OOMPH_EXCEPTION_LOCATION);
3056  }
3057  }
3058  }
3059 #endif
3060  // ...and point to the existing one
3061  finite_ext_el_pt->node_pt(j) = existing_node_pt;
3062  }
3063  // If it doesn't add it to the list of existing ones
3064  else
3065  {
3066  global_node_pt[first_non_negative_eqn_number_plus_one - 1] =
3067  nod_pt;
3068  node_done[nod_pt] = true;
3069  }
3070  }
3071 
3072 
3073  // Do the same for any master nodes of that (possibly replaced)
3074  // node
3075  if (dynamic_cast<RefineableElement*>(finite_ext_el_pt) != 0)
3076  {
3077  int n_cont_inter_values =
3078  dynamic_cast<RefineableElement*>(finite_ext_el_pt)
3079  ->ncont_interpolated_values();
3080  for (int i_cont = -1; i_cont < n_cont_inter_values; i_cont++)
3081  {
3082  if (finite_ext_el_pt->node_pt(j)->is_hanging(i_cont))
3083  {
3084  HangInfo* hang_pt =
3085  finite_ext_el_pt->node_pt(j)->hanging_pt(i_cont);
3086  unsigned n_master = hang_pt->nmaster();
3087  for (unsigned m = 0; m < n_master; m++)
3088  {
3089  Node* master_nod_pt = hang_pt->master_node_pt(m);
3090  unsigned n_val = master_nod_pt->nvalue();
3091  unsigned first_non_negative_eqn_number_plus_one = 0;
3092  for (unsigned i_val = 0; i_val < n_val; i_val++)
3093  {
3094  int eqn_no = master_nod_pt->eqn_number(i_val);
3095  if (eqn_no >= 0)
3096  {
3097  first_non_negative_eqn_number_plus_one = eqn_no + 1;
3098  break;
3099  }
3100  }
3101 
3102  // If we haven't found a non-negative eqn number check
3103  // eqn numbers associated with solid data (if any)
3104  if (first_non_negative_eqn_number_plus_one == 0)
3105  {
3106  SolidNode* solid_master_nod_pt =
3107  dynamic_cast<SolidNode*>(master_nod_pt);
3108  if (solid_master_nod_pt != 0)
3109  {
3110  // Loop over values stored at node (if any) to find
3111  // the first non-negative eqn number
3112  unsigned n_val =
3113  solid_master_nod_pt->variable_position_pt()
3114  ->nvalue();
3115  for (unsigned i_val = 0; i_val < n_val; i_val++)
3116  {
3117  int eqn_no =
3118  solid_master_nod_pt->variable_position_pt()
3119  ->eqn_number(i_val);
3120  if (eqn_no >= 0)
3121  {
3122  first_non_negative_eqn_number_plus_one =
3123  eqn_no + 1;
3124  break;
3125  }
3126  }
3127  }
3128  }
3129 
3130  // Identified which node we're dealing with via
3131  // first non-negative global eqn number (if there
3132  // is none, everything is pinned and we don't give a
3133  // damn...)
3134  if (first_non_negative_eqn_number_plus_one > 0)
3135  {
3136  Node* existing_node_pt = global_node_pt
3137  [first_non_negative_eqn_number_plus_one - 1];
3138 
3139  // Does this node already exist?
3140  if (existing_node_pt != 0)
3141  {
3142  // Record that we're about to cull one
3143  actually_removed_some_data = true;
3144 
3145  // It's a duplicate, so store the duplicated one for
3146  // later killing...
3147  Node* duplicated_node_pt = master_nod_pt;
3148 
3149  if (!node_done[duplicated_node_pt])
3150  {
3151  // Remove node from all boundaries
3152  std::set<unsigned>* boundaries_pt;
3153  duplicated_node_pt->get_boundaries_pt(
3154  boundaries_pt);
3155  if (boundaries_pt != 0)
3156  {
3157  for (std::set<unsigned>::iterator it =
3158  (*boundaries_pt).begin();
3159  it != (*boundaries_pt).end();
3160  it++)
3161  {
3163  (*it), duplicated_node_pt);
3164  }
3165  }
3166 
3167  killed_nodes.insert(duplicated_node_pt);
3168  unsigned i_proc = unsigned(iproc);
3170  i_proc, duplicated_node_pt);
3171  }
3172 
3173  // Weight of the original node
3174  double m_weight = hang_pt->master_weight(m);
3175 
3176 
3177 #ifdef PARANOID
3178  // Sanity check: setting replacement master
3179  // node for non-hanging node? Sign of really
3180  // f***ed up code.
3181  Node* tmp_nod_pt = finite_ext_el_pt->node_pt(j);
3182  if (!tmp_nod_pt->is_hanging(i_cont))
3183  {
3184  std::ostringstream error_stream;
3185  error_stream
3186  << "About to re-set master for i_cont= " << i_cont
3187  << " for external node (with proc " << iproc
3188  << " )" << tmp_nod_pt << " at ";
3189  unsigned n = tmp_nod_pt->ndim();
3190  for (unsigned jj = 0; jj < n; jj++)
3191  {
3192  error_stream << tmp_nod_pt->x(jj) << " ";
3193  }
3194  error_stream
3195  << " which is not hanging --> About to die!"
3196  << "Outputting offending element into oomph-info "
3197  << "stream. \n\n";
3198  oomph_info << "\n\n";
3199  finite_ext_el_pt->output(*(oomph_info.stream_pt()));
3200  oomph_info << "\n\n";
3201  oomph_info.stream_pt()->flush();
3202  throw OomphLibError(error_stream.str(),
3203  OOMPH_CURRENT_FUNCTION,
3204  OOMPH_EXCEPTION_LOCATION);
3205  }
3206 #endif
3207 
3208 
3209  // And re-set master
3210  finite_ext_el_pt->node_pt(j)
3211  ->hanging_pt(i_cont)
3212  ->set_master_node_pt(m, existing_node_pt, m_weight);
3213  }
3214  // If it doesn't, add it to the list of existing ones
3215  else
3216  {
3217  global_node_pt
3218  [first_non_negative_eqn_number_plus_one - 1] =
3219  master_nod_pt;
3220  node_done[master_nod_pt] = true;
3221  }
3222  }
3223  } // End of loop over master nodes
3224  } // end of hanging
3225  } // end of loop over continously interpolated variables
3226  } // end refineable element (with potentially hanging node
3227 
3228  } // end loop over nodes on external halo elements
3229 
3230  } // End of check for finite element
3231 
3232  } // end loop over external halo elements
3233  }
3234  } // end loop over processors
3235 
3236 
3237  // Now kill all the deleted nodes
3238  for (std::set<Node*>::iterator it = killed_nodes.begin();
3239  it != killed_nodes.end();
3240  it++)
3241  {
3242  delete (*it);
3243  }
3244 
3245 
3246  // oomph_info << "Number of nonzero entries in global_node_pt: "
3247  // << global_node_pt.size() << std::endl;
3248 
3249  // // Time it...
3250  // // Taken out again by MH -- clutters up output
3251  // if (Global_timings::Doc_comprehensive_timings)
3252  // {
3253  // double t_end = TimingHelpers::timer();
3254  // oomph_info
3255  // << "Total time for Problem::remove_duplicate_data: "
3256  // << t_end-t_start << std::endl;
3257  // }
3258  }
3259 
3260 
3261  //========================================================================
3262  /// Consolidate external halo node storage by removing nulled out
3263  /// pointers in external halo and haloed schemes for all meshes.
3264  //========================================================================
3266  {
3267  // Do we have submeshes?
3268  unsigned n_mesh_loop = 1;
3269  unsigned nmesh = nsub_mesh();
3270  if (nmesh > 0)
3271  {
3272  n_mesh_loop = nmesh;
3273  }
3274 
3275  // Storage for number of processors and current processor
3276  int n_proc = this->communicator_pt()->nproc();
3277  int my_rank = this->communicator_pt()->my_rank();
3278 
3279  // If only one processor then return
3280  if (n_proc == 1)
3281  {
3282  return;
3283  }
3284 
3285  // Loop over all (other) processors and store index of any nulled-out
3286  // external halo nodes in storage scheme.
3287 
3288  // Data to be sent to each processor
3289  Vector<int> send_n(n_proc, 0);
3290 
3291  // Storage for all values to be sent to all processors
3292  Vector<int> send_data;
3293 
3294  // Start location within send_data for data to be sent to each processor
3295  Vector<int> send_displacement(n_proc, 0);
3296 
3297  // Check missing ones
3298  for (int domain = 0; domain < n_proc; domain++)
3299  {
3300  // Set the offset for the current processor
3301  send_displacement[domain] = send_data.size();
3302 
3303  // Don't bother to do anything if the processor in the loop is the
3304  // current processor
3305  if (domain != my_rank)
3306  {
3307  // Deal with sub-meshes one-by-one if required
3308  Mesh* my_mesh_pt = 0;
3309 
3310  // Loop over submeshes
3311  for (unsigned imesh = 0; imesh < n_mesh_loop; imesh++)
3312  {
3313  if (nmesh == 0)
3314  {
3315  my_mesh_pt = mesh_pt();
3316  }
3317  else
3318  {
3319  my_mesh_pt = mesh_pt(imesh);
3320  }
3321 
3322  // Make backup of external halo node pointers with this domain
3323  Vector<Node*> backup_pt(my_mesh_pt->external_halo_node_pt(domain));
3324 
3325  // How many do we have currently?
3326  unsigned nnod = backup_pt.size();
3327 
3328  // Prepare storage for updated halo nodes
3329  Vector<Node*> new_external_halo_node_pt;
3330  new_external_halo_node_pt.reserve(nnod);
3331 
3332  // Loop over external halo nodes with this domain
3333  for (unsigned j = 0; j < nnod; j++)
3334  {
3335  // Get pointer to node
3336  Node* nod_pt = backup_pt[j];
3337 
3338  // Has it been nulled out?
3339  if (nod_pt == 0)
3340  {
3341  // Save index of nulled out one
3342  send_data.push_back(j);
3343  }
3344  else
3345  {
3346  // Still alive: Copy across
3347  new_external_halo_node_pt.push_back(nod_pt);
3348  }
3349  }
3350 
3351  // Set new external halo node vector
3352  my_mesh_pt->set_external_halo_node_pt(domain,
3353  new_external_halo_node_pt);
3354 
3355  // End of data for this mesh
3356  send_data.push_back(-1);
3357 
3358  } // end of loop over meshes
3359 
3360  } // end skip own domain
3361 
3362  // Find the number of data added to the vector
3363  send_n[domain] = send_data.size() - send_displacement[domain];
3364  }
3365 
3366  // Storage for the number of data to be received from each processor
3367  Vector<int> receive_n(n_proc, 0);
3368 
3369  // Now send numbers of data to be sent between all processors
3370  MPI_Alltoall(&send_n[0],
3371  1,
3372  MPI_INT,
3373  &receive_n[0],
3374  1,
3375  MPI_INT,
3376  this->communicator_pt()->mpi_comm());
3377 
3378 
3379  // We now prepare the data to be received
3380  // by working out the displacements from the received data
3381  Vector<int> receive_displacement(n_proc, 0);
3382  int receive_data_count = 0;
3383  for (int rank = 0; rank < n_proc; ++rank)
3384  {
3385  // Displacement is number of data received so far
3386  receive_displacement[rank] = receive_data_count;
3387  receive_data_count += receive_n[rank];
3388  }
3389 
3390  // Now resize the receive buffer for all data from all processors
3391  // Make sure that it has a size of at least one
3392  if (receive_data_count == 0)
3393  {
3394  ++receive_data_count;
3395  }
3396  Vector<int> receive_data(receive_data_count);
3397 
3398  // Make sure that the send buffer has size at least one
3399  // so that we don't get a segmentation fault
3400  if (send_data.size() == 0)
3401  {
3402  send_data.resize(1);
3403  }
3404 
3405  // Now send the data between all the processors
3406  MPI_Alltoallv(&send_data[0],
3407  &send_n[0],
3408  &send_displacement[0],
3409  MPI_INT,
3410  &receive_data[0],
3411  &receive_n[0],
3412  &receive_displacement[0],
3413  MPI_INT,
3414  this->communicator_pt()->mpi_comm());
3415 
3416  // Now use the received data
3417  for (int send_rank = 0; send_rank < n_proc; send_rank++)
3418  {
3419  // Don't bother to do anything for the processor corresponding to the
3420  // current processor or if no data were received from this processor
3421  if ((send_rank != my_rank) && (receive_n[send_rank] != 0))
3422  {
3423  // Counter for the data within the large array
3424  unsigned count = receive_displacement[send_rank];
3425 
3426  // Deal with sub-meshes one-by-one if required
3427  Mesh* my_mesh_pt = 0;
3428 
3429  // Loop over submeshes
3430  for (unsigned imesh = 0; imesh < n_mesh_loop; imesh++)
3431  {
3432  if (nmesh == 0)
3433  {
3434  my_mesh_pt = mesh_pt();
3435  }
3436  else
3437  {
3438  my_mesh_pt = mesh_pt(imesh);
3439  }
3440 
3441  // Make backup of external haloed node pointers with this domain
3442  Vector<Node*> backup_pt =
3443  my_mesh_pt->external_haloed_node_pt(send_rank);
3444 
3445  // Unpack until we reach "end of data" indicator (-1) for this mesh
3446  while (true)
3447  {
3448  // Read next entry
3449  int next_one = receive_data[count++];
3450 
3451  if (next_one == -1)
3452  {
3453  break;
3454  }
3455  else
3456  {
3457  // Null out the entry
3458  backup_pt[next_one] = 0;
3459  }
3460  }
3461 
3462  // How many do we have currently?
3463  unsigned nnod = backup_pt.size();
3464 
3465  // Prepare storage for updated haloed nodes
3466  Vector<Node*> new_external_haloed_node_pt;
3467  new_external_haloed_node_pt.reserve(nnod);
3468 
3469  // Loop over external haloed nodes with this domain
3470  for (unsigned j = 0; j < nnod; j++)
3471  {
3472  // Get pointer to node
3473  Node* nod_pt = backup_pt[j];
3474 
3475  // Has it been nulled out?
3476  if (nod_pt != 0)
3477  {
3478  // Still alive: Copy across
3479  new_external_haloed_node_pt.push_back(nod_pt);
3480  }
3481  }
3482 
3483  // Set new external haloed node vector
3484  my_mesh_pt->set_external_haloed_node_pt(send_rank,
3485  new_external_haloed_node_pt);
3486  }
3487  }
3488 
3489  } // End of data is received
3490  }
3491 
3492 #endif
3493 
3494 
3495  //=======================================================================
3496  /// Function that sets the values of the dofs in the object
3497  //======================================================================
3499  {
3500  const unsigned long n_dof = this->ndof();
3501 #ifdef PARANOID
3502  if (n_dof != dofs.nrow())
3503  {
3504  std::ostringstream error_stream;
3505  error_stream << "Number of degrees of freedom in vector argument "
3506  << dofs.nrow() << "\n"
3507  << "does not equal number of degrees of freedom in problem "
3508  << n_dof;
3509  throw OomphLibError(
3510  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
3511  }
3512 #endif
3513  for (unsigned long l = 0; l < n_dof; l++)
3514  {
3515  *Dof_pt[l] = dofs[l];
3516  }
3517  }
3518 
3519  /// Set history values of dofs
3520  void Problem::set_dofs(const unsigned& t, DoubleVector& dofs)
3521  {
3522 #ifdef PARANOID
3523  if (distributed())
3524  {
3525  throw OomphLibError("Not designed for distributed problems",
3526  OOMPH_EXCEPTION_LOCATION,
3527  OOMPH_CURRENT_FUNCTION);
3528  // might work if the dofs vector is distributed in the right way...
3529  }
3530 #endif
3531 
3532  // First deal with global data
3533  unsigned Nglobal_data = nglobal_data();
3534  for (unsigned i = 0; i < Nglobal_data; i++)
3535  {
3536  for (unsigned j = 0, nj = Global_data_pt[i]->nvalue(); j < nj; j++)
3537  {
3538  // For each data get the equation number and copy out the value.
3539  int eqn_number = Global_data_pt[i]->eqn_number(j);
3540  if (eqn_number >= 0)
3541  {
3542  Global_data_pt[i]->set_value(t, j, dofs[eqn_number]);
3543  }
3544  }
3545  }
3546 
3547  // Next element internal data
3548  for (unsigned i = 0, ni = mesh_pt()->nelement(); i < ni; i++)
3549  {
3550  GeneralisedElement* ele_pt = mesh_pt()->element_pt(i);
3551  for (unsigned j = 0, nj = ele_pt->ninternal_data(); j < nj; j++)
3552  {
3553  Data* d_pt = ele_pt->internal_data_pt(j);
3554  for (unsigned k = 0, nk = d_pt->nvalue(); k < nk; k++)
3555  {
3556  int eqn_number = d_pt->eqn_number(k);
3557  if (eqn_number >= 0)
3558  {
3559  d_pt->set_value(t, k, dofs[eqn_number]);
3560  }
3561  }
3562  }
3563  }
3564 
3565  // Now the nodes
3566  for (unsigned i = 0, ni = mesh_pt()->nnode(); i < ni; i++)
3567  {
3568  Node* node_pt = mesh_pt()->node_pt(i);
3569  for (unsigned j = 0, nj = node_pt->nvalue(); j < nj; j++)
3570  {
3571  // For each node get the equation number and copy out the value.
3572  int eqn_number = node_pt->eqn_number(j);
3573  if (eqn_number >= 0)
3574  {
3575  node_pt->set_value(t, j, dofs[eqn_number]);
3576  }
3577  }
3578  }
3579  }
3580 
3581 
3582  /// Set history values of dofs from the type of vector stored in
3583  /// problem::Dof_pt.
3584  void Problem::set_dofs(const unsigned& t, Vector<double*>& dof_pt)
3585  {
3586 #ifdef PARANOID
3587  if (distributed())
3588  {
3589  throw OomphLibError("Not implemented for distributed problems!",
3590  OOMPH_EXCEPTION_LOCATION,
3591  OOMPH_CURRENT_FUNCTION);
3592  }
3593 #endif
3594 
3595  // If we have any spine meshes I think there might be more degrees
3596  // of freedom there. I don't use them though so I'll let someone who
3597  // knows what they are doing handle it. --David Shepherd
3598 
3599  // First deal with global data
3600  unsigned Nglobal_data = nglobal_data();
3601  for (unsigned i = 0; i < Nglobal_data; i++)
3602  {
3603  for (unsigned j = 0, nj = Global_data_pt[i]->nvalue(); j < nj; j++)
3604  {
3605  // For each data get the equation number and copy in the value.
3606  int eqn_number = Global_data_pt[i]->eqn_number(j);
3607  if (eqn_number >= 0)
3608  {
3609  Global_data_pt[i]->set_value(t, j, *(dof_pt[eqn_number]));
3610  }
3611  }
3612  }
3613 
3614  // Now the mesh data
3615  // nodes
3616  for (unsigned i = 0, ni = mesh_pt()->nnode(); i < ni; i++)
3617  {
3618  Node* node_pt = mesh_pt()->node_pt(i);
3619  for (unsigned j = 0, nj = node_pt->nvalue(); j < nj; j++)
3620  {
3621  // For each node get the equation number and copy in the value.
3622  int eqn_number = node_pt->eqn_number(j);
3623  if (eqn_number >= 0)
3624  {
3625  node_pt->set_value(t, j, *(dof_pt[eqn_number]));
3626  }
3627  }
3628  }
3629 
3630  // and non-nodal data inside elements
3631  for (unsigned i = 0, ni = mesh_pt()->nelement(); i < ni; i++)
3632  {
3633  GeneralisedElement* ele_pt = mesh_pt()->element_pt(i);
3634  for (unsigned j = 0, nj = ele_pt->ninternal_data(); j < nj; j++)
3635  {
3636  Data* data_pt = ele_pt->internal_data_pt(j);
3637  // For each node get the equation number and copy in the value.
3638  int eqn_number = data_pt->eqn_number(j);
3639  if (eqn_number >= 0)
3640  {
3641  data_pt->set_value(t, j, *(dof_pt[eqn_number]));
3642  }
3643  }
3644  }
3645  }
3646 
3647 
3648  //===================================================================
3649  /// Function that adds the values to the dofs
3650  //==================================================================
3651  void Problem::add_to_dofs(const double& lambda,
3652  const DoubleVector& increment_dofs)
3653  {
3654  const unsigned long n_dof = this->ndof();
3655  for (unsigned long l = 0; l < n_dof; l++)
3656  {
3657  *Dof_pt[l] += lambda * increment_dofs[l];
3658  }
3659  }
3660 
3661 
3662  //=========================================================================
3663  /// Return the residual vector multiplied by the inverse mass matrix
3664  /// Virtual so that it can be overloaded for mpi problems
3665  //=========================================================================
3667  {
3668  // This function does not make sense for assembly handlers other than the
3669  // default, so complain if we try to call it with another handler
3670 
3671 #ifdef PARANOID
3672  // If we are not the default, then complain
3674  {
3675  std::ostringstream error_stream;
3676  error_stream << "The function get_inverse_mass_matrix_times_residuals() "
3677  "can only be\n"
3678  << "used with the default assembly handler\n\n";
3679  throw OomphLibError(
3680  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
3681  }
3682 #endif
3683 
3684  // Find the number of degrees of freedom in the problem
3685  const unsigned n_dof = this->ndof();
3686 
3687  // Resize the vector
3688  LinearAlgebraDistribution dist(this->communicator_pt(), n_dof, false);
3689  Mres.build(&dist, 0.0);
3690 
3691  // If we have discontinuous formulation
3692  // We can invert the mass matrix element by element
3694  {
3695  // Loop over the elements and get their residuals
3696  const unsigned n_element = Problem::mesh_pt()->nelement();
3697  Vector<double> element_Mres;
3698  for (unsigned e = 0; e < n_element; e++)
3699  {
3700  // Cache the element
3701  DGElement* const elem_pt =
3702  dynamic_cast<DGElement*>(Problem::mesh_pt()->element_pt(e));
3703 
3704  // Find the elemental inverse mass matrix times residuals
3705  const unsigned n_el_dofs = elem_pt->ndof();
3706  elem_pt->get_inverse_mass_matrix_times_residuals(element_Mres);
3707 
3708  // Add contribution to global matrix
3709  for (unsigned i = 0; i < n_el_dofs; i++)
3710  {
3711  Mres[elem_pt->eqn_number(i)] = element_Mres[i];
3712  }
3713  }
3714  }
3715  // Otherwise it's continous and we must invert the full
3716  // mass matrix via a global linear solve.
3717  else
3718  {
3719  // Now do the linear solve -- recycling Mass matrix if requested
3720  // If we already have the factorised mass matrix, then resolve
3722  {
3724  {
3725  oomph_info << "Not recomputing Mass Matrix " << std::endl;
3726  }
3727 
3728  // Get the residuals
3729  DoubleVector residuals(&dist, 0.0);
3730  this->get_residuals(residuals);
3731 
3732  // Resolve the linear system
3734  residuals, Mres);
3735  }
3736  // Otherwise solve for the first time
3737  else
3738  {
3739  // If we wish to reuse the mass matrix, then enable resolve
3741  {
3743  {
3744  oomph_info << "Enabling resolve in explicit timestep" << std::endl;
3745  }
3747  ->enable_resolve();
3748  }
3749 
3750  // Use a custom assembly handler to assemble and invert the mass matrix
3751 
3752  // Store the old assembly handler
3753  AssemblyHandler* old_assembly_handler_pt = this->assembly_handler_pt();
3754  // Set the assembly handler to the explicit timestep handler
3756 
3757  // Solve the linear system
3759  Mres);
3760  // The mass matrix has now been computed
3762 
3763  // Delete the Explicit Timestep handler
3764  delete this->assembly_handler_pt();
3765  // Reset the assembly handler to the original handler
3766  this->assembly_handler_pt() = old_assembly_handler_pt;
3767  }
3768  }
3769  }
3770 
3772  {
3773  // Loop over timesteppers: make them (temporarily) steady and store their
3774  // is_steady status.
3775  unsigned n_time_steppers = this->ntime_stepper();
3776  std::vector<bool> was_steady(n_time_steppers);
3777  for (unsigned i = 0; i < n_time_steppers; i++)
3778  {
3779  was_steady[i] = time_stepper_pt(i)->is_steady();
3781  }
3782 
3783  // Calculate f using the residual/jacobian machinary.
3785 
3786  // Reset the is_steady status of all timesteppers that weren't already
3787  // steady when we came in here and reset their weights
3788  for (unsigned i = 0; i < n_time_steppers; i++)
3789  {
3790  if (!was_steady[i])
3791  {
3793  }
3794  }
3795  }
3796 
3797 
3798  //================================================================
3799  /// Get the total residuals Vector for the problem
3800  //================================================================
3802  {
3803  // Three different cases; if MPI_Helpers::MPI_has_been_initialised=true
3804  // this means MPI_Helpers::init() has been called. This could happen on a
3805  // code compiled with MPI but run serially; in this instance the
3806  // get_residuals function still works on one processor.
3807  //
3808  // Secondly, if a code has been compiled with MPI, but MPI_Helpers::init()
3809  // has not been called, then MPI_Helpers::MPI_has_been_initialised=false
3810  // and the code calls...
3811  //
3812  // Thirdly, the serial version (compiled by all, but only run when compiled
3813  // with MPI if MPI_Helpers::MPI_has_been_initialised=false
3814 
3815  // Check that the residuals has the correct number of rows if it has been
3816  // setup
3817 #ifdef PARANOID
3818  if (residuals.built())
3819  {
3820  if (residuals.distribution_pt()->nrow() != this->ndof())
3821  {
3822  std::ostringstream error_stream;
3823  error_stream << "The distribution of the residuals vector does not "
3824  "have the correct\n"
3825  << "number of global rows\n";
3826 
3827  throw OomphLibError(
3828  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
3829  }
3830  }
3831 #endif
3832 
3833  // Determine the distribution for the residuals vector
3834  // IF the vector has distribution setup then use that
3835  // ELSE determine the distribution based on the
3836  // distributed_matrix_distribution enum
3837  LinearAlgebraDistribution* dist_pt = 0;
3838  if (residuals.built())
3839  {
3840  dist_pt = new LinearAlgebraDistribution(residuals.distribution_pt());
3841  }
3842  else
3843  {
3845  }
3846 
3847  // Locally cache pointer to assembly handler
3849 
3850  // Build and zero the residuals
3851  residuals.build(dist_pt, 0.0);
3852 
3853  // Serial (or one processor case)
3854 #ifdef OOMPH_HAS_MPI
3855  if (this->communicator_pt()->nproc() == 1)
3856  {
3857 #endif // OOMPH_HAS_MPI
3858  // Loop over all the elements
3859  unsigned long Element_pt_range = Mesh_pt->nelement();
3860  for (unsigned long e = 0; e < Element_pt_range; e++)
3861  {
3862  // Get the pointer to the element
3863  GeneralisedElement* elem_pt = Mesh_pt->element_pt(e);
3864  // Find number of dofs in the element
3865  unsigned n_element_dofs = assembly_handler_pt->ndof(elem_pt);
3866  // Set up an array
3867  Vector<double> element_residuals(n_element_dofs);
3868  // Fill the array
3869  assembly_handler_pt->get_residuals(elem_pt, element_residuals);
3870  // Now loop over the dofs and assign values to global Vector
3871  for (unsigned l = 0; l < n_element_dofs; l++)
3872  {
3873  residuals[assembly_handler_pt->eqn_number(elem_pt, l)] +=
3874  element_residuals[l];
3875  }
3876  }
3877  // Otherwise parallel case
3878 #ifdef OOMPH_HAS_MPI
3879  }
3880  else
3881  {
3882  // Store the current assembly handler
3883  AssemblyHandler* const old_assembly_handler_pt = Assembly_handler_pt;
3884  // Create a new assembly handler that only assembles the residuals
3886  new ParallelResidualsHandler(old_assembly_handler_pt);
3887 
3888  // Setup memory for parallel sparse assemble
3889  // No matrix so all size zero
3890  Vector<int*> column_index;
3891  Vector<int*> row_start;
3892  Vector<double*> value;
3893  Vector<unsigned> nnz;
3894  // One set of residuals of sizer one
3895  Vector<double*> res(1);
3896 
3897  // Call the parallel sparse assemble, that should only assemble residuals
3899  dist_pt, column_index, row_start, value, nnz, res);
3900  // Fill in the residuals data
3901  residuals.set_external_values(res[0], true);
3902 
3903  // Delete new assembly handler
3904  delete Assembly_handler_pt;
3905  // Reset the assembly handler to the original
3906  Assembly_handler_pt = old_assembly_handler_pt;
3907  }
3908 #endif
3909 
3910  // Delete the distribution
3911  delete dist_pt;
3912  }
3913 
3914  //=============================================================================
3915  /// Get the fully assembled residual vector and Jacobian matrix
3916  /// in dense storage. The DoubleVector residuals returned will be
3917  /// non-distributed. If on calling this method the DoubleVector residuals is
3918  /// setup then it must be non-distributed and of the correct length.
3919  /// The matrix type DenseDoubleMatrix is not distributable and therefore
3920  /// the residual vector is also assumed to be non distributable.
3921  //=============================================================================
3923  DenseDoubleMatrix& jacobian)
3924  {
3925  // get the number of degrees of freedom
3926  unsigned n_dof = ndof();
3927 
3928 #ifdef PARANOID
3929  // PARANOID checks : if the distribution of residuals is setup then it must
3930  // must not be distributed, have the right number of rows, and the same
3931  // communicator as the problem
3932  if (residuals.built())
3933  {
3934  if (residuals.distribution_pt()->distributed())
3935  {
3936  std::ostringstream error_stream;
3937  error_stream
3938  << "If the DoubleVector residuals is setup then it must not "
3939  << "be distributed.";
3940  throw OomphLibError(
3941  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
3942  }
3943  if (residuals.distribution_pt()->nrow() != n_dof)
3944  {
3945  std::ostringstream error_stream;
3946  error_stream
3947  << "If the DoubleVector residuals is setup then it must have"
3948  << " the correct number of rows";
3949  throw OomphLibError(
3950  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
3951  }
3952  if (!(*Communicator_pt ==
3953  *residuals.distribution_pt()->communicator_pt()))
3954  {
3955  std::ostringstream error_stream;
3956  error_stream
3957  << "If the DoubleVector residuals is setup then it must have"
3958  << " the same communicator as the problem.";
3959  throw OomphLibError(
3960  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
3961  }
3962  }
3963 #endif
3964 
3965  // set the residuals distribution if it is not setup
3966  if (!residuals.built())
3967  {
3968  LinearAlgebraDistribution dist(Communicator_pt, n_dof, false);
3969  residuals.build(&dist, 0.0);
3970  }
3971  // else just zero the residuals
3972  else
3973  {
3974  residuals.initialise(0.0);
3975  }
3976 
3977  // Resize the matrices -- this cannot always be done externally
3978  // because get_jacobian exists in many different versions for
3979  // different storage formats -- resizing a CC or CR matrix doesn't
3980  // make sense.
3981 
3982  // resize the jacobian
3983  jacobian.resize(n_dof, n_dof);
3984  jacobian.initialise(0.0);
3985 
3986  // Locally cache pointer to assembly handler
3988 
3989  // Loop over all the elements
3990  unsigned long n_element = Mesh_pt->nelement();
3991  for (unsigned long e = 0; e < n_element; e++)
3992  {
3993  // Get the pointer to the element
3994  GeneralisedElement* elem_pt = Mesh_pt->element_pt(e);
3995  // Find number of dofs in the element
3996  unsigned n_element_dofs = assembly_handler_pt->ndof(elem_pt);
3997  // Set up an array
3998  Vector<double> element_residuals(n_element_dofs);
3999  // Set up a matrix
4000  DenseMatrix<double> element_jacobian(n_element_dofs);
4001  // Fill the array
4003  elem_pt, element_residuals, element_jacobian);
4004  // Now loop over the dofs and assign values to global Vector
4005  for (unsigned l = 0; l < n_element_dofs; l++)
4006  {
4007  unsigned long eqn_number = assembly_handler_pt->eqn_number(elem_pt, l);
4008  residuals[eqn_number] += element_residuals[l];
4009  for (unsigned l2 = 0; l2 < n_element_dofs; l2++)
4010  {
4011  jacobian(eqn_number, assembly_handler_pt->eqn_number(elem_pt, l2)) +=
4012  element_jacobian(l, l2);
4013  }
4014  }
4015  }
4016  }
4017 
4018  //=============================================================================
4019  /// Return the fully-assembled Jacobian and residuals for the problem,
4020  /// in the case where the Jacobian matrix is in a distributable
4021  /// row compressed storage format.
4022  /// 1. If the distribution of the jacobian and residuals is setup then, they
4023  /// will be returned with that distribution.
4024  /// Note. the jacobian and residuals must have the same distribution.
4025  /// 2. If the distribution of the jacobian and residuals are not setup then
4026  /// their distribution will computed based on:
4027  /// Distributed_problem_matrix_distribution.
4028  //=============================================================================
4030  {
4031  // Three different cases; if MPI_Helpers::MPI_has_been_initialised=true
4032  // this means MPI_Helpers::setup() has been called. This could happen on a
4033  // code compiled with MPI but run serially; in this instance the
4034  // get_residuals function still works on one processor.
4035  //
4036  // Secondly, if a code has been compiled with MPI, but MPI_Helpers::setup()
4037  // has not been called, then MPI_Helpers::MPI_has_been_initialised=false
4038  // and the code calls...
4039  //
4040  // Thirdly, the serial version (compiled by all, but only run when compiled
4041  // with MPI if MPI_Helpers::MPI_has_been_initialised=false
4042  //
4043  // The only case where an MPI code cannot run serially at present
4044  // is one where the distribute function is used (i.e. METIS is called)
4045 
4046  // Allocate storage for the matrix entries
4047  // The generalised Vector<Vector<>> structure is required
4048  // for the most general interface to sparse_assemble() which allows
4049  // the assembly of multiple matrices at once.
4050  Vector<int*> column_index(1);
4051  Vector<int*> row_start(1);
4052  Vector<double*> value(1);
4053  Vector<unsigned> nnz(1);
4054 
4055 #ifdef PARANOID
4056  // PARANOID checks that the distribution of the jacobian matches that of the
4057  // residuals (if they are setup) and that they have the right number of rows
4058  if (residuals.built() && jacobian.distribution_built())
4059  {
4060  if (!(*residuals.distribution_pt() == *jacobian.distribution_pt()))
4061  {
4062  std::ostringstream error_stream;
4063  error_stream << "The distribution of the residuals must "
4064  << "be the same as the distribution of the jacobian.";
4065  throw OomphLibError(
4066  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
4067  }
4068  if (jacobian.distribution_pt()->nrow() != this->ndof())
4069  {
4070  std::ostringstream error_stream;
4071  error_stream
4072  << "The distribution of the jacobian and residuals does not"
4073  << "have the correct number of global rows.";
4074  throw OomphLibError(
4075  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
4076  }
4077  }
4078  else if (residuals.built() != jacobian.distribution_built())
4079  {
4080  std::ostringstream error_stream;
4081  error_stream << "The distribution of the jacobian and residuals must "
4082  << "both be setup or both not setup";
4083  throw OomphLibError(
4084  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
4085  }
4086 #endif
4087 
4088 
4089  // Allocate generalised storage format for passing to sparse_assemble()
4090  Vector<double*> res(1);
4091 
4092  // determine the distribution for the jacobian.
4093  // IF the jacobian has distribution setup then use that
4094  // ELSE determine the distribution based on the
4095  // distributed_matrix_distribution enum
4096  LinearAlgebraDistribution* dist_pt = 0;
4097  if (jacobian.distribution_built())
4098  {
4099  dist_pt = new LinearAlgebraDistribution(jacobian.distribution_pt());
4100  }
4101  else
4102  {
4104  }
4105 
4106 
4107  // The matrix is in compressed row format
4108  bool compressed_row_flag = true;
4109 
4110 #ifdef OOMPH_HAS_MPI
4111  //
4112  if (Communicator_pt->nproc() == 1)
4113  {
4114 #endif
4116  column_index, row_start, value, nnz, res, compressed_row_flag);
4117  jacobian.build(dist_pt);
4118  jacobian.build_without_copy(
4119  dist_pt->nrow(), nnz[0], value[0], column_index[0], row_start[0]);
4120  residuals.build(dist_pt, 0.0);
4121  residuals.set_external_values(res[0], true);
4122 #ifdef OOMPH_HAS_MPI
4123  }
4124  else
4125  {
4126  if (dist_pt->distributed())
4127  {
4129  dist_pt, column_index, row_start, value, nnz, res);
4130  jacobian.build(dist_pt);
4131  jacobian.build_without_copy(
4132  dist_pt->nrow(), nnz[0], value[0], column_index[0], row_start[0]);
4133  residuals.build(dist_pt, 0.0);
4134  residuals.set_external_values(res[0], true);
4135  }
4136  else
4137  {
4138  LinearAlgebraDistribution* temp_dist_pt =
4139  new LinearAlgebraDistribution(Communicator_pt, dist_pt->nrow(), true);
4141  temp_dist_pt, column_index, row_start, value, nnz, res);
4142  jacobian.build(temp_dist_pt);
4143  jacobian.build_without_copy(
4144  dist_pt->nrow(), nnz[0], value[0], column_index[0], row_start[0]);
4145  jacobian.redistribute(dist_pt);
4146  residuals.build(temp_dist_pt, 0.0);
4147  residuals.set_external_values(res[0], true);
4148  residuals.redistribute(dist_pt);
4149  delete temp_dist_pt;
4150  }
4151  }
4152 #endif
4153 
4154  // clean up dist_pt and residuals_vector pt
4155  delete dist_pt;
4156  }
4157 
4158  //=============================================================================
4159  /// Return the fully-assembled Jacobian and residuals for the problem,
4160  /// in the case when the jacobian matrix is in column-compressed storage
4161  /// format.
4162  //=============================================================================
4164  {
4165  // Three different cases; if MPI_Helpers::MPI_has_been_initialised=true
4166  // this means MPI_Helpers::setup() has been called. This could happen on a
4167  // code compiled with MPI but run serially; in this instance the
4168  // get_residuals function still works on one processor.
4169  //
4170  // Secondly, if a code has been compiled with MPI, but MPI_Helpers::setup()
4171  // has not been called, then MPI_Helpers::MPI_has_been_initialised=false
4172  // and the code calls...
4173  //
4174  // Thirdly, the serial version (compiled by all, but only run when compiled
4175  // with MPI if MPI_Helpers::MPI_has_been_5Binitialised=false
4176  //
4177  // The only case where an MPI code cannot run serially at present
4178  // is one where the distribute function is used (i.e. METIS is called)
4179 
4180  // get the number of degrees of freedom
4181  unsigned n_dof = ndof();
4182 
4183 #ifdef PARANOID
4184  // PARANOID checks : if the distribution of residuals is setup then it must
4185  // must not be distributed, have the right number of rows, and the same
4186  // communicator as the problem
4187  if (residuals.built())
4188  {
4189  if (residuals.distribution_pt()->distributed())
4190  {
4191  std::ostringstream error_stream;
4192  error_stream
4193  << "If the DoubleVector residuals is setup then it must not "
4194  << "be distributed.";
4195  throw OomphLibError(
4196  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
4197  }
4198  if (residuals.distribution_pt()->nrow() != n_dof)
4199  {
4200  std::ostringstream error_stream;
4201  error_stream
4202  << "If the DoubleVector residuals is setup then it must have"
4203  << " the correct number of rows";
4204  throw OomphLibError(
4205  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
4206  }
4207  if (!(*Communicator_pt ==
4208  *residuals.distribution_pt()->communicator_pt()))
4209  {
4210  std::ostringstream error_stream;
4211  error_stream
4212  << "If the DoubleVector residuals is setup then it must have"
4213  << " the same communicator as the problem.";
4214  throw OomphLibError(
4215  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
4216  }
4217  }
4218 #endif
4219 
4220  // Allocate storage for the matrix entries
4221  // The generalised Vector<Vector<>> structure is required
4222  // for the most general interface to sparse_assemble() which allows
4223  // the assembly of multiple matrices at once.
4224  Vector<int*> row_index(1);
4225  Vector<int*> column_start(1);
4226  Vector<double*> value(1);
4227 
4228  // Allocate generalised storage format for passing to sparse_assemble()
4229  Vector<double*> res(1);
4230 
4231  // allocate storage for the number of non-zeros in each matrix
4232  Vector<unsigned> nnz(1);
4233 
4234  // The matrix is in compressed column format
4235  bool compressed_row_flag = false;
4236 
4237  // get the distribution for the residuals
4238  LinearAlgebraDistribution* dist_pt;
4239  if (!residuals.built())
4240  {
4241  dist_pt =
4242  new LinearAlgebraDistribution(Communicator_pt, this->ndof(), false);
4243  }
4244  else
4245  {
4246  dist_pt = new LinearAlgebraDistribution(residuals.distribution_pt());
4247  }
4248 
4249 #ifdef OOMPH_HAS_MPI
4250  if (communicator_pt()->nproc() == 1)
4251  {
4252 #endif
4254  row_index, column_start, value, nnz, res, compressed_row_flag);
4255  jacobian.build_without_copy(
4256  value[0], row_index[0], column_start[0], nnz[0], n_dof, n_dof);
4257  residuals.build(dist_pt, 0.0);
4258  residuals.set_external_values(res[0], true);
4259 #ifdef OOMPH_HAS_MPI
4260  }
4261  else
4262  {
4263  std::ostringstream error_stream;
4264  error_stream << "Cannot assemble a CCDoubleMatrix Jacobian on more "
4265  << "than one processor.";
4266  throw OomphLibError(
4267  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
4268  }
4269 #endif
4270 
4271  // clean up
4272  delete dist_pt;
4273  }
4274 
4275 
4276  //===================================================================
4277  /// Set all pinned values to zero.
4278  /// Used to set boundary conditions to be homogeneous in the copy
4279  /// of the problem used in adaptive bifurcation tracking
4280  /// (ALH: TEMPORARY HACK, WILL BE FIXED)
4281  //==================================================================
4283  {
4284  // NOTE THIS DOES NOT ZERO ANY SPINE DATA, but otherwise everything else
4285  // should be zeroed
4286 
4287  // Zero any pinned global Data
4288  const unsigned n_global_data = nglobal_data();
4289  for (unsigned i = 0; i < n_global_data; i++)
4290  {
4291  Data* const local_data_pt = Global_data_pt[i];
4292  const unsigned n_value = local_data_pt->nvalue();
4293  for (unsigned j = 0; j < n_value; j++)
4294  {
4295  // If the data value is pinned set the value to zero
4296  if (local_data_pt->is_pinned(j))
4297  {
4298  local_data_pt->set_value(j, 0.0);
4299  }
4300  }
4301  }
4302 
4303  // Loop over the submeshes:
4304  const unsigned n_sub_mesh = Sub_mesh_pt.size();
4305  if (n_sub_mesh == 0)
4306  {
4307  // Loop over the nodes in the element
4308  const unsigned n_node = Mesh_pt->nnode();
4309  for (unsigned n = 0; n < n_node; n++)
4310  {
4311  Node* const local_node_pt = Mesh_pt->node_pt(n);
4312  const unsigned n_value = local_node_pt->nvalue();
4313  for (unsigned j = 0; j < n_value; j++)
4314  {
4315  // If the data value is pinned set the value to zero
4316  if (local_node_pt->is_pinned(j))
4317  {
4318  local_node_pt->set_value(j, 0.0);
4319  }
4320  }
4321 
4322  // Try to cast to a solid node
4323  SolidNode* const local_solid_node_pt =
4324  dynamic_cast<SolidNode*>(local_node_pt);
4325  // If we are successful
4326  if (local_solid_node_pt)
4327  {
4328  // Find the dimension of the node
4329  const unsigned n_dim = local_solid_node_pt->ndim();
4330  // Find number of positions
4331  const unsigned n_position_type =
4332  local_solid_node_pt->nposition_type();
4333 
4334  for (unsigned k = 0; k < n_position_type; k++)
4335  {
4336  for (unsigned i = 0; i < n_dim; i++)
4337  {
4338  // If the generalised position is pinned,
4339  // set the value to zero
4340  if (local_solid_node_pt->position_is_pinned(k, i))
4341  {
4342  local_solid_node_pt->x_gen(k, i) = 0.0;
4343  }
4344  }
4345  }
4346  }
4347  }
4348 
4349  // Now loop over the element's and zero the internal data
4350  const unsigned n_element = Mesh_pt->nelement();
4351  for (unsigned e = 0; e < n_element; e++)
4352  {
4353  GeneralisedElement* const local_element_pt = Mesh_pt->element_pt(e);
4354  const unsigned n_internal = local_element_pt->ninternal_data();
4355  for (unsigned i = 0; i < n_internal; i++)
4356  {
4357  Data* const local_data_pt = local_element_pt->internal_data_pt(i);
4358  const unsigned n_value = local_data_pt->nvalue();
4359  for (unsigned j = 0; j < n_value; j++)
4360  {
4361  // If the data value is pinned set the value to zero
4362  if (local_data_pt->is_pinned(j))
4363  {
4364  local_data_pt->set_value(j, 0.0);
4365  }
4366  }
4367  }
4368  } // End of loop over elements
4369  }
4370  else
4371  {
4372  // Alternatively loop over all sub meshes
4373  for (unsigned m = 0; m < n_sub_mesh; m++)
4374  {
4375  // Loop over the nodes in the element
4376  const unsigned n_node = Sub_mesh_pt[m]->nnode();
4377  for (unsigned n = 0; n < n_node; n++)
4378  {
4379  Node* const local_node_pt = Sub_mesh_pt[m]->node_pt(n);
4380  const unsigned n_value = local_node_pt->nvalue();
4381  for (unsigned j = 0; j < n_value; j++)
4382  {
4383  // If the data value is pinned set the value to zero
4384  if (local_node_pt->is_pinned(j))
4385  {
4386  local_node_pt->set_value(j, 0.0);
4387  }
4388  }
4389 
4390  // Try to cast to a solid node
4391  SolidNode* const local_solid_node_pt =
4392  dynamic_cast<SolidNode*>(local_node_pt);
4393  // If we are successful
4394  if (local_solid_node_pt)
4395  {
4396  // Find the dimension of the node
4397  const unsigned n_dim = local_solid_node_pt->ndim();
4398  // Find number of positions
4399  const unsigned n_position_type =
4400  local_solid_node_pt->nposition_type();
4401 
4402  for (unsigned k = 0; k < n_position_type; k++)
4403  {
4404  for (unsigned i = 0; i < n_dim; i++)
4405  {
4406  // If the generalised position is pinned,
4407  // set the value to zero
4408  if (local_solid_node_pt->position_is_pinned(k, i))
4409  {
4410  local_solid_node_pt->x_gen(k, i) = 0.0;
4411  }
4412  }
4413  }
4414  }
4415  }
4416 
4417  // Now loop over the element's and zero the internal data
4418  const unsigned n_element = Sub_mesh_pt[m]->nelement();
4419  for (unsigned e = 0; e < n_element; e++)
4420  {
4421  GeneralisedElement* const local_element_pt =
4422  Sub_mesh_pt[m]->element_pt(e);
4423  const unsigned n_internal = local_element_pt->ninternal_data();
4424  for (unsigned i = 0; i < n_internal; i++)
4425  {
4426  Data* const local_data_pt = local_element_pt->internal_data_pt(i);
4427  const unsigned n_value = local_data_pt->nvalue();
4428  for (unsigned j = 0; j < n_value; j++)
4429  {
4430  // If the data value is pinned set the value to zero
4431  if (local_data_pt->is_pinned(j))
4432  {
4433  local_data_pt->set_value(j, 0.0);
4434  }
4435  }
4436  }
4437  } // End of loop over elements
4438  }
4439  }
4440  }
4441 
4442 
4443  //=====================================================================
4444  /// This is a (private) helper function that is used to assemble system
4445  /// matrices in compressed row or column format
4446  /// and compute residual vectors.
4447  /// The default action is to assemble the jacobian matrix and
4448  /// residuals for the Newton method. The action can be
4449  /// overloaded at an elemental level by changing the default
4450  /// behaviour of the function Element::get_all_vectors_and_matrices().
4451  /// column_or_row_index: Column [or row] index of given entry
4452  /// row_or_column_start: Index of first entry for given row [or column]
4453  /// value : Vector of nonzero entries
4454  /// residuals : Residual vector
4455  /// compressed_row_flag: Bool flag to indicate if storage format is
4456  /// compressed row [if false interpretation of
4457  /// arguments is as stated in square brackets].
4458  /// We provide four different assembly methods, each with different
4459  /// memory requirements/execution speeds. The method is set by
4460  /// the public flag Problem::Sparse_assembly_method.
4461  //=====================================================================
4463  Vector<int*>& column_or_row_index,
4464  Vector<int*>& row_or_column_start,
4465  Vector<double*>& value,
4466  Vector<unsigned>& nnz,
4467  Vector<double*>& residuals,
4468  bool compressed_row_flag)
4469  {
4470  // Choose the actual method
4471  switch (Sparse_assembly_method)
4472  {
4474 
4476  column_or_row_index,
4477  row_or_column_start,
4478  value,
4479  nnz,
4480  residuals,
4481  compressed_row_flag);
4482 
4483  break;
4484 
4486 
4488  column_or_row_index,
4489  row_or_column_start,
4490  value,
4491  nnz,
4492  residuals,
4493  compressed_row_flag);
4494 
4495  break;
4496 
4498 
4500  row_or_column_start,
4501  value,
4502  nnz,
4503  residuals,
4504  compressed_row_flag);
4505 
4506  break;
4507 
4509 
4511  column_or_row_index,
4512  row_or_column_start,
4513  value,
4514  nnz,
4515  residuals,
4516  compressed_row_flag);
4517 
4518  break;
4519 
4521 
4523  column_or_row_index,
4524  row_or_column_start,
4525  value,
4526  nnz,
4527  residuals,
4528  compressed_row_flag);
4529 
4530  break;
4531 
4532  default:
4533 
4534  std::ostringstream error_stream;
4535  error_stream
4536  << "Error: Incorrect value for Problem::Sparse_assembly_method"
4537  << Sparse_assembly_method << std::endl
4538  << "It should be one of the enumeration Problem::Assembly_method"
4539  << std::endl;
4540  throw OomphLibError(
4541  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
4542  }
4543  }
4544 
4545 
4546  //=====================================================================
4547  /// This is a (private) helper function that is used to assemble system
4548  /// matrices in compressed row or column format
4549  /// and compute residual vectors, using maps
4550  /// The default action is to assemble the jacobian matrix and
4551  /// residuals for the Newton method. The action can be
4552  /// overloaded at an elemental level by chaging the default
4553  /// behaviour of the function Element::get_all_vectors_and_matrices().
4554  /// column_or_row_index: Column [or row] index of given entry
4555  /// row_or_column_start: Index of first entry for given row [or column]
4556  /// value : Vector of nonzero entries
4557  /// residuals : Residual vector
4558  /// compressed_row_flag: Bool flag to indicate if storage format is
4559  /// compressed row [if false interpretation of
4560  /// arguments is as stated in square brackets].
4561  //=====================================================================
4563  Vector<int*>& column_or_row_index,
4564  Vector<int*>& row_or_column_start,
4565  Vector<double*>& value,
4566  Vector<unsigned>& nnz,
4567  Vector<double*>& residuals,
4568  bool compressed_row_flag)
4569  {
4570  // Total number of elements
4571  const unsigned long n_elements = mesh_pt()->nelement();
4572 
4573  // Default range of elements for distributed problems
4574  unsigned long el_lo = 0;
4575  unsigned long el_hi = n_elements - 1;
4576 
4577 #ifdef OOMPH_HAS_MPI
4578  // Otherwise just loop over a fraction of the elements
4579  // (This will either have been initialised in
4580  // Problem::set_default_first_and_last_element_for_assembly() or
4581  // will have been re-assigned during a previous assembly loop
4582  // Note that following the re-assignment only the entries
4583  // for the current processor are relevant.
4585  {
4586  el_lo = First_el_for_assembly[Communicator_pt->my_rank()];
4587  el_hi = Last_el_plus_one_for_assembly[Communicator_pt->my_rank()] - 1;
4588  }
4589 #endif
4590 
4591  // number of dofs
4592  unsigned ndof = this->ndof();
4593 
4594  // Find the number of vectors to be assembled
4595  const unsigned n_vector = residuals.size();
4596 
4597  // Find the number of matrices to be assembled
4598  const unsigned n_matrix = column_or_row_index.size();
4599 
4600  // Locally cache pointer to assembly handler
4602 
4603 #ifdef OOMPH_HAS_MPI
4604  bool doing_residuals = false;
4605  if (dynamic_cast<ParallelResidualsHandler*>(Assembly_handler_pt) != 0)
4606  {
4607  doing_residuals = true;
4608  }
4609 #endif
4610 
4611 // Error check dimensions
4612 #ifdef PARANOID
4613  if (row_or_column_start.size() != n_matrix)
4614  {
4615  std::ostringstream error_stream;
4616  error_stream << "Error: " << std::endl
4617  << "row_or_column_start.size() "
4618  << row_or_column_start.size() << " does not equal "
4619  << "column_or_row_index.size() "
4620  << column_or_row_index.size() << std::endl;
4621  throw OomphLibError(
4622  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
4623  }
4624 
4625  if (value.size() != n_matrix)
4626  {
4627  std::ostringstream error_stream;
4628  error_stream
4629  << "Error in Problem::sparse_assemble_row_or_column_compressed "
4630  << std::endl
4631  << "value.size() " << value.size() << " does not equal "
4632  << "column_or_row_index.size() " << column_or_row_index.size()
4633  << std::endl
4634  << std::endl
4635  << std::endl;
4636  throw OomphLibError(
4637  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
4638  }
4639 #endif
4640 
4641 
4642  // The idea behind this sparse assembly routine is to use a vector of
4643  // maps for the entries in each row or column of the complete matrix.
4644  // The key for each map is the global row or column number and
4645  // the default comparison operator for integers means that each map
4646  // is ordered by the global row or column number. Thus, we need not
4647  // sort the maps, that happens at each insertion of a new entry. The
4648  // price we pay is that for large maps, inseration is not a
4649  // cheap operation. Hash maps can be used to increase the speed, but then
4650  // the ordering is lost and we would have to sort anyway. The solution if
4651  // speed is required is to use lists, see below.
4652 
4653 
4654  // Set up a vector of vectors of maps of entries of each matrix,
4655  // indexed by either the column or row. The entries of the vector for
4656  // each matrix correspond to all the rows or columns of that matrix.
4657  // The use of the map storage
4658  // scheme, with its implicit ordering on the first index, gives
4659  // a sparse ordered list of the entries in the given row or column.
4660  Vector<Vector<std::map<unsigned, double>>> matrix_data_map(n_matrix);
4661  // Loop over the number of matrices being assembled and resize
4662  // each vector of maps to the number of rows or columns of the matrix
4663  for (unsigned m = 0; m < n_matrix; m++)
4664  {
4665  matrix_data_map[m].resize(ndof);
4666  }
4667 
4668  // Resize the residuals vectors
4669  for (unsigned v = 0; v < n_vector; v++)
4670  {
4671  residuals[v] = new double[ndof];
4672  for (unsigned i = 0; i < ndof; i++)
4673  {
4674  residuals[v][i] = 0;
4675  }
4676  }
4677 
4678 
4679 #ifdef OOMPH_HAS_MPI
4680 
4681 
4682  // Storage for assembly time for elements
4683  double t_assemble_start = 0.0;
4684 
4685  // Storage for assembly times
4686  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
4687  {
4688  Elemental_assembly_time.resize(n_elements);
4689  }
4690 
4691 #endif
4692 
4693  //----------------Assemble and populate the maps-------------------------
4694  {
4695  // Allocate local storage for the element's contribution to the
4696  // residuals vectors and system matrices of the size of the maximum
4697  // number of dofs in any element.
4698  // This means that the storage is only allocated (and deleted) once
4699  Vector<Vector<double>> el_residuals(n_vector);
4700  Vector<DenseMatrix<double>> el_jacobian(n_matrix);
4701 
4702  // Loop over the elements for this processor
4703  for (unsigned long e = el_lo; e <= el_hi; e++)
4704  {
4705 #ifdef OOMPH_HAS_MPI
4706  // Time it?
4707  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
4708  {
4709  t_assemble_start = TimingHelpers::timer();
4710  }
4711 #endif
4712 
4713  // Get the pointer to the element
4714  GeneralisedElement* elem_pt = mesh_pt()->element_pt(e);
4715 
4716 #ifdef OOMPH_HAS_MPI
4717  // Ignore halo elements
4718  if (!elem_pt->is_halo())
4719  {
4720 #endif
4721 
4722  // Find number of degrees of freedom in the element
4723  const unsigned nvar = assembly_handler_pt->ndof(elem_pt);
4724 
4725  // Resize the storage for elemental jacobian and residuals
4726  for (unsigned v = 0; v < n_vector; v++)
4727  {
4728  el_residuals[v].resize(nvar);
4729  }
4730  for (unsigned m = 0; m < n_matrix; m++)
4731  {
4732  el_jacobian[m].resize(nvar);
4733  }
4734 
4735  // Now get the residuals and jacobian for the element
4737  elem_pt, el_residuals, el_jacobian);
4738 
4739  //---------------Insert the values into the maps--------------
4740 
4741  // Loop over the first index of local variables
4742  for (unsigned i = 0; i < nvar; i++)
4743  {
4744  // Get the local equation number
4745  unsigned eqn_number = assembly_handler_pt->eqn_number(elem_pt, i);
4746 
4747  // Add the contribution to the residuals
4748  for (unsigned v = 0; v < n_vector; v++)
4749  {
4750  // Fill in each residuals vector
4751  residuals[v][eqn_number] += el_residuals[v][i];
4752  }
4753 
4754  // Now loop over the other index
4755  for (unsigned j = 0; j < nvar; j++)
4756  {
4757  // Get the number of the unknown
4758  unsigned unknown = assembly_handler_pt->eqn_number(elem_pt, j);
4759 
4760  // Loop over the matrices
4761  for (unsigned m = 0; m < n_matrix; m++)
4762  {
4763  // Get the value of the matrix at this point
4764  double value = el_jacobian[m](i, j);
4765  // Only bother to add to the map if it's non-zero
4766  if (std::fabs(value) > Numerical_zero_for_sparse_assembly)
4767  {
4768  // If it's compressed row storage, then our vector of maps
4769  // is indexed by row (equation number)
4770  if (compressed_row_flag)
4771  {
4772  // Add the data into the map using the unknown as the map
4773  // key
4774  matrix_data_map[m][eqn_number][unknown] += value;
4775  }
4776  // Otherwise it's compressed column storage and our vector is
4777  // indexed by column (the unknown)
4778  else
4779  {
4780  // Add the data into the map using the eqn_numbe as the map
4781  // key
4782  matrix_data_map[m][unknown][eqn_number] += value;
4783  }
4784  }
4785  } // End of loop over matrices
4786  }
4787  }
4788 
4789 #ifdef OOMPH_HAS_MPI
4790  } // endif halo element
4791 #endif
4792 
4793 
4794 #ifdef OOMPH_HAS_MPI
4795  // Time it?
4796  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
4797  {
4799  TimingHelpers::timer() - t_assemble_start;
4800  }
4801 #endif
4802 
4803  } // End of loop over the elements
4804 
4805  } // End of map assembly
4806 
4807 
4808 #ifdef OOMPH_HAS_MPI
4809 
4810  // Postprocess timing information and re-allocate distribution of
4811  // elements during subsequent assemblies.
4812  if ((!doing_residuals) && (!Problem_has_been_distributed) &&
4814  {
4816  }
4817 
4818  // We have determined load balancing for current setup.
4819  // This can remain the same until assign_eqn_numbers() is called
4820  // again -- the flag is re-set to true there.
4821  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
4822  {
4824  }
4825 
4826 #endif
4827 
4828 
4829  //-----------Finally we need to convert the beautiful map storage scheme
4830  //------------------------to the containers required by SuperLU
4831 
4832  // Loop over the number of matrices
4833  for (unsigned m = 0; m < n_matrix; m++)
4834  {
4835  // Set the number of rows or columns
4836  row_or_column_start[m] = new int[ndof + 1];
4837  // Counter for the total number of entries in the storage scheme
4838  unsigned long entry_count = 0;
4839  row_or_column_start[m][0] = entry_count;
4840 
4841  // first we compute the number of non-zeros
4842  nnz[m] = 0;
4843  for (unsigned long i_global = 0; i_global < ndof; i_global++)
4844  {
4845  nnz[m] += matrix_data_map[m][i_global].size();
4846  }
4847 
4848  // and then resize the storage
4849  column_or_row_index[m] = new int[nnz[m]];
4850  value[m] = new double[nnz[m]];
4851 
4852  // Now we merely loop over the number of rows or columns
4853  for (unsigned long i_global = 0; i_global < ndof; i_global++)
4854  {
4855  // Start index for the present row
4856  row_or_column_start[m][i_global] = entry_count;
4857  // If there are no entries in the map then skip the rest of the loop
4858  if (matrix_data_map[m][i_global].empty())
4859  {
4860  continue;
4861  }
4862 
4863  // Loop over all the entries in the map corresponding to the given
4864  // row or column. It will be ordered
4865 
4866  for (std::map<unsigned, double>::iterator it =
4867  matrix_data_map[m][i_global].begin();
4868  it != matrix_data_map[m][i_global].end();
4869  ++it)
4870  {
4871  // The first value is the column or row index
4872  column_or_row_index[m][entry_count] = it->first;
4873  // The second value is the actual data value
4874  value[m][entry_count] = it->second;
4875  // Increase the value of the counter
4876  entry_count++;
4877  }
4878  }
4879 
4880  // Final entry in the row/column start vector
4881  row_or_column_start[m][ndof] = entry_count;
4882  } // End of the loop over the matrices
4883 
4885  {
4886  oomph_info << "Pausing at end of sparse assembly." << std::endl;
4887  pause("Check memory usage now.");
4888  }
4889  }
4890 
4891 
4892  //=====================================================================
4893  /// This is a (private) helper function that is used to assemble system
4894  /// matrices in compressed row or column format
4895  /// and compute residual vectors using lists
4896  /// The default action is to assemble the jacobian matrix and
4897  /// residuals for the Newton method. The action can be
4898  /// overloaded at an elemental level by chaging the default
4899  /// behaviour of the function Element::get_all_vectors_and_matrices().
4900  /// column_or_row_index: Column [or row] index of given entry
4901  /// row_or_column_start: Index of first entry for given row [or column]
4902  /// value : Vector of nonzero entries
4903  /// residuals : Residual vector
4904  /// compressed_row_flag: Bool flag to indicate if storage format is
4905  /// compressed row [if false interpretation of
4906  /// arguments is as stated in square brackets].
4907  //=====================================================================
4909  Vector<int*>& column_or_row_index,
4910  Vector<int*>& row_or_column_start,
4911  Vector<double*>& value,
4912  Vector<unsigned>& nnz,
4913  Vector<double*>& residuals,
4914  bool compressed_row_flag)
4915  {
4916  // Total number of elements
4917  const unsigned long n_elements = mesh_pt()->nelement();
4918 
4919  // Default range of elements for distributed problems
4920  unsigned long el_lo = 0;
4921  unsigned long el_hi = n_elements - 1;
4922 
4923 #ifdef OOMPH_HAS_MPI
4924  // Otherwise just loop over a fraction of the elements
4925  // (This will either have been initialised in
4926  // Problem::set_default_first_and_last_element_for_assembly() or
4927  // will have been re-assigned during a previous assembly loop
4928  // Note that following the re-assignment only the entries
4929  // for the current processor are relevant.
4931  {
4932  el_lo = First_el_for_assembly[Communicator_pt->my_rank()];
4933  el_hi = Last_el_plus_one_for_assembly[Communicator_pt->my_rank()] - 1;
4934  }
4935 #endif
4936 
4937  // number of dofs
4938  unsigned ndof = this->ndof();
4939 
4940  // Find the number of vectors to be assembled
4941  const unsigned n_vector = residuals.size();
4942 
4943  // Find the number of matrices to be assembled
4944  const unsigned n_matrix = column_or_row_index.size();
4945 
4946  // Locally cache pointer to assembly handler
4948 
4949 #ifdef OOMPH_HAS_MPI
4950  bool doing_residuals = false;
4951  if (dynamic_cast<ParallelResidualsHandler*>(Assembly_handler_pt) != 0)
4952  {
4953  doing_residuals = true;
4954  }
4955 #endif
4956 
4957 // Error check dimensions
4958 #ifdef PARANOID
4959  if (row_or_column_start.size() != n_matrix)
4960  {
4961  std::ostringstream error_stream;
4962  error_stream << "Error: " << std::endl
4963  << "row_or_column_start.size() "
4964  << row_or_column_start.size() << " does not equal "
4965  << "column_or_row_index.size() "
4966  << column_or_row_index.size() << std::endl;
4967  throw OomphLibError(
4968  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
4969  }
4970 
4971  if (value.size() != n_matrix)
4972  {
4973  std::ostringstream error_stream;
4974  error_stream
4975  << "Error in Problem::sparse_assemble_row_or_column_compressed "
4976  << std::endl
4977  << "value.size() " << value.size() << " does not equal "
4978  << "column_or_row_index.size() " << column_or_row_index.size()
4979  << std::endl
4980  << std::endl
4981  << std::endl;
4982  throw OomphLibError(
4983  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
4984  }
4985 #endif
4986 
4987  // The idea behind this sparse assembly routine is to use a vector of
4988  // lists for the entries in each row or column of the complete matrix.
4989  // The lists contain pairs of entries (global row/column number, value).
4990  // All non-zero contributions from each element are added to the lists.
4991  // We then sort each list by global row/column number and then combine
4992  // the entries corresponding to each row/column before adding to the
4993  // vectors column_or_row_index and value.
4994 
4995  // Note the trade off for "fast assembly" is that we will require
4996  // more memory during the assembly phase. Then again, if we can
4997  // only just assemble the sparse matrix, we're in real trouble.
4998 
4999  // Set up a vector of lists of paired entries of
5000  //(row/column index, jacobian matrix entry).
5001  // The entries of the vector correspond to all the rows or columns.
5002  // The use of the list storage scheme, should give fast insertion
5003  // and fast sorts later.
5005  n_matrix);
5006  // Loop over the number of matrices and resize
5007  for (unsigned m = 0; m < n_matrix; m++)
5008  {
5009  matrix_data_list[m].resize(ndof);
5010  }
5011 
5012  // Resize the residuals vectors
5013  for (unsigned v = 0; v < n_vector; v++)
5014  {
5015  residuals[v] = new double[ndof];
5016  for (unsigned i = 0; i < ndof; i++)
5017  {
5018  residuals[v][i] = 0;
5019  }
5020  }
5021 
5022 #ifdef OOMPH_HAS_MPI
5023 
5024 
5025  // Storage for assembly time for elements
5026  double t_assemble_start = 0.0;
5027 
5028  // Storage for assembly times
5029  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
5030  {
5031  Elemental_assembly_time.resize(n_elements);
5032  }
5033 
5034 #endif
5035 
5036  //------------Assemble and populate the lists-----------------------
5037  {
5038  // Allocate local storage for the element's contribution to the
5039  // residuals vectors and system matrices of the size of the maximum
5040  // number of dofs in any element.
5041  // This means that the stored is only allocated (and deleted) once
5042  Vector<Vector<double>> el_residuals(n_vector);
5043  Vector<DenseMatrix<double>> el_jacobian(n_matrix);
5044 
5045 
5046  // Pointer to a single list to be used during the assembly
5047  std::list<std::pair<unsigned, double>>* list_pt;
5048 
5049  // Loop over the all elements
5050  for (unsigned long e = el_lo; e <= el_hi; e++)
5051  {
5052 #ifdef OOMPH_HAS_MPI
5053  // Time it?
5054  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
5055  {
5056  t_assemble_start = TimingHelpers::timer();
5057  }
5058 #endif
5059 
5060  // Get the pointer to the element
5061  GeneralisedElement* elem_pt = mesh_pt()->element_pt(e);
5062 
5063 #ifdef OOMPH_HAS_MPI
5064  // Ignore halo elements
5065  if (!elem_pt->is_halo())
5066  {
5067 #endif
5068 
5069  // Find number of degrees of freedom in the element
5070  const unsigned nvar = assembly_handler_pt->ndof(elem_pt);
5071 
5072  // Resize the storage for the elemental jacobian and residuals
5073  for (unsigned v = 0; v < n_vector; v++)
5074  {
5075  el_residuals[v].resize(nvar);
5076  }
5077  for (unsigned m = 0; m < n_matrix; m++)
5078  {
5079  el_jacobian[m].resize(nvar);
5080  }
5081 
5082  // Now get the residuals and jacobian for the element
5084  elem_pt, el_residuals, el_jacobian);
5085 
5086  //---------------- Insert the values into the lists -----------
5087 
5088  // Loop over the first index of local variables
5089  for (unsigned i = 0; i < nvar; i++)
5090  {
5091  // Get the local equation number
5092  unsigned eqn_number = assembly_handler_pt->eqn_number(elem_pt, i);
5093 
5094  // Add the contribution to the residuals
5095  for (unsigned v = 0; v < n_vector; v++)
5096  {
5097  // Fill in the residuals vector
5098  residuals[v][eqn_number] += el_residuals[v][i];
5099  }
5100 
5101  // Now loop over the other index
5102  for (unsigned j = 0; j < nvar; j++)
5103  {
5104  // Get the number of the unknown
5105  unsigned unknown = assembly_handler_pt->eqn_number(elem_pt, j);
5106 
5107  // Loop over the matrices
5108  for (unsigned m = 0; m < n_matrix; m++)
5109  {
5110  // Get the value of the matrix at this point
5111  double value = el_jacobian[m](i, j);
5112  // Only add to theif it's non-zero
5113  if (std::fabs(value) > Numerical_zero_for_sparse_assembly)
5114  {
5115  // If it's compressed row storage, then our vector is indexed
5116  // by row (the equation number)
5117  if (compressed_row_flag)
5118  {
5119  // Find the list that corresponds to the desired row
5120  list_pt = &matrix_data_list[m][eqn_number];
5121  // Insert the data into the list, the first entry
5122  // in the pair is the unknown (column index),
5123  // the second is the value itself.
5124  list_pt->insert(list_pt->end(),
5125  std::make_pair(unknown, value));
5126  }
5127  // Otherwise it's compressed column storage, and our
5128  // vector is indexed by column (the unknown)
5129  else
5130  {
5131  // Find the list that corresponds to the desired column
5132  list_pt = &matrix_data_list[m][unknown];
5133  // Insert the data into the list, the first entry
5134  // in the pair is the equation number (row index),
5135  // the second is the value itself.
5136  list_pt->insert(list_pt->end(),
5137  std::make_pair(eqn_number, value));
5138  }
5139  }
5140  }
5141  }
5142  }
5143 
5144 #ifdef OOMPH_HAS_MPI
5145  } // endif halo element
5146 #endif
5147 
5148 
5149 #ifdef OOMPH_HAS_MPI
5150  // Time it?
5151  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
5152  {
5154  TimingHelpers::timer() - t_assemble_start;
5155  }
5156 #endif
5157 
5158  } // End of loop over the elements
5159 
5160  } // list_pt goes out of scope
5161 
5162 
5163 #ifdef OOMPH_HAS_MPI
5164 
5165  // Postprocess timing information and re-allocate distribution of
5166  // elements during subsequent assemblies.
5167  if ((!doing_residuals) && (!Problem_has_been_distributed) &&
5169  {
5171  }
5172 
5173  // We have determined load balancing for current setup.
5174  // This can remain the same until assign_eqn_numbers() is called
5175  // again -- the flag is re-set to true there.
5176  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
5177  {
5179  }
5180 
5181 #endif
5182 
5183 
5184  //----Finally we need to convert the beautiful list storage scheme---
5185  //----------to the containers required by SuperLU--------------------
5186 
5187  // Loop over the number of matrices
5188  for (unsigned m = 0; m < n_matrix; m++)
5189  {
5190  // Set the number of rows or columns
5191  row_or_column_start[m] = new int[ndof + 1];
5192  // Counter for the total number of entries in the storage scheme
5193  unsigned long entry_count = 0;
5194  // The first entry is 0
5195  row_or_column_start[m][0] = entry_count;
5196 
5197  // first we compute the number of non-zeros
5198  nnz[m] = 0;
5199  for (unsigned long i_global = 0; i_global < ndof; i_global++)
5200  {
5201  nnz[m] += matrix_data_list[m][i_global].size();
5202  }
5203 
5204  // and then resize the storage
5205  column_or_row_index[m] = new int[nnz[m]];
5206  value[m] = new double[nnz[m]];
5207 
5208  // Now we merely loop over the number of rows or columns
5209  for (unsigned long i_global = 0; i_global < ndof; i_global++)
5210  {
5211  // Start index for the present row is the number of entries so far
5212  row_or_column_start[m][i_global] = entry_count;
5213  // If there are no entries in the list then skip the loop
5214  if (matrix_data_list[m][i_global].empty())
5215  {
5216  continue;
5217  }
5218 
5219  // Sort the list corresponding to this row or column by the
5220  // column or row index (first entry in the pair).
5221  // This might be inefficient, but we only have to do the sort ONCE
5222  // for each list. This is faster than using a map storage scheme, where
5223  // we are sorting for every insertion (although the map structure
5224  // is cleaner and more memory efficient)
5225  matrix_data_list[m][i_global].sort();
5226 
5227  // Set up an iterator for start of the list
5228  std::list<std::pair<unsigned, double>>::iterator it =
5229  matrix_data_list[m][i_global].begin();
5230 
5231  // Get the first row or column index in the list...
5232  unsigned current_index = it->first;
5233  //...and the corresponding value
5234  double current_value = it->second;
5235 
5236  // Loop over all the entries in the sorted list
5237  // Increase the iterator so that we start at the second entry
5238  for (++it; it != matrix_data_list[m][i_global].end(); ++it)
5239  {
5240  // If the index has not changed, then we must add the contribution
5241  // of the present entry to the value.
5242  // Additionally check that the entry is non-zero
5243  if ((it->first == current_index) &&
5244  (std::fabs(it->second) > Numerical_zero_for_sparse_assembly))
5245  {
5246  current_value += it->second;
5247  }
5248  // Otherwise, we have added all the contributions to the index
5249  // to current_value, so add it to the SuperLU data structure
5250  else
5251  {
5252  // Add the row or column index to the vector
5253  column_or_row_index[m][entry_count] = current_index;
5254  // Add the actual value to the vector
5255  value[m][entry_count] = current_value;
5256  // Increase the counter for the number of entries in each vector
5257  entry_count++;
5258 
5259  // Set the index and value to be those of the current entry in the
5260  // list
5261  current_index = it->first;
5262  current_value = it->second;
5263  }
5264  } // End of loop over all list entries for this global row or column
5265 
5266  // There are TWO special cases to consider.
5267  // If there is only one equation number in the list, then it
5268  // will NOT have been added. We test this case by comparing the
5269  // number of entries with those stored in row_or_column_start[i_global]
5270  // Otherwise
5271  // If the final entry in the list has the same index as the penultimate
5272  // entry, then it will NOT have been added to the SuperLU storage scheme
5273  // Check this by comparing the current_index with the final index
5274  // stored in the SuperLU scheme. If they are not the same, then
5275  // add the current_index and value.
5276 
5277  // If single equation number in list
5278  if ((static_cast<int>(entry_count) == row_or_column_start[m][i_global])
5279  // If we have a single equation number, this will not be evaluated.
5280  // If we don't then we do the test to check that the final
5281  // entry is added
5282  || (static_cast<int>(current_index) !=
5283  column_or_row_index[m][entry_count - 1]))
5284  {
5285  // Add the row or column index to the vector
5286  column_or_row_index[m][entry_count] = current_index;
5287  // Add the actual value to the vector
5288  value[m][entry_count] = current_value;
5289  // Increase the counter for the number of entries in each vector
5290  entry_count++;
5291  }
5292 
5293  } // End of loop over the rows or columns of the entire matrix
5294 
5295  // Final entry in the row/column start vector
5296  row_or_column_start[m][ndof] = entry_count;
5297  } // End of loop over matrices
5298 
5300  {
5301  oomph_info << "Pausing at end of sparse assembly." << std::endl;
5302  pause("Check memory usage now.");
5303  }
5304  }
5305 
5306 
5307  //=====================================================================
5308  /// This is a (private) helper function that is used to assemble system
5309  /// matrices in compressed row or column format
5310  /// and compute residual vectors using vectors of pairs
5311  /// The default action is to assemble the jacobian matrix and
5312  /// residuals for the Newton method. The action can be
5313  /// overloaded at an elemental level by chaging the default
5314  /// behaviour of the function Element::get_all_vectors_and_matrices().
5315  /// column_or_row_index: Column [or row] index of given entry
5316  /// row_or_column_start: Index of first entry for given row [or column]
5317  /// value : Vector of nonzero entries
5318  /// residuals : Residual vector
5319  /// compressed_row_flag: Bool flag to indicate if storage format is
5320  /// compressed row [if false interpretation of
5321  /// arguments is as stated in square brackets].
5322  //=====================================================================
5324  Vector<int*>& column_or_row_index,
5325  Vector<int*>& row_or_column_start,
5326  Vector<double*>& value,
5327  Vector<unsigned>& nnz,
5328  Vector<double*>& residuals,
5329  bool compressed_row_flag)
5330  {
5331  // Total number of elements
5332  const unsigned long n_elements = mesh_pt()->nelement();
5333 
5334  // Default range of elements for distributed problems
5335  unsigned long el_lo = 0;
5336  unsigned long el_hi = n_elements - 1;
5337 
5338 #ifdef OOMPH_HAS_MPI
5339  // Otherwise just loop over a fraction of the elements
5340  // (This will either have been initialised in
5341  // Problem::set_default_first_and_last_element_for_assembly() or
5342  // will have been re-assigned during a previous assembly loop
5343  // Note that following the re-assignment only the entries
5344  // for the current processor are relevant.
5346  {
5347  el_lo = First_el_for_assembly[Communicator_pt->my_rank()];
5348  el_hi = Last_el_plus_one_for_assembly[Communicator_pt->my_rank()] - 1;
5349  }
5350 #endif
5351 
5352  // number of local eqns
5353  unsigned ndof = this->ndof();
5354 
5355  // Find the number of vectors to be assembled
5356  const unsigned n_vector = residuals.size();
5357 
5358  // Find the number of matrices to be assembled
5359  const unsigned n_matrix = column_or_row_index.size();
5360 
5361  // Locally cache pointer to assembly handler
5363 
5364 #ifdef OOMPH_HAS_MPI
5365  bool doing_residuals = false;
5366  if (dynamic_cast<ParallelResidualsHandler*>(Assembly_handler_pt) != 0)
5367  {
5368  doing_residuals = true;
5369  }
5370 #endif
5371 
5372 // Error check dimensions
5373 #ifdef PARANOID
5374  if (row_or_column_start.size() != n_matrix)
5375  {
5376  std::ostringstream error_stream;
5377  error_stream << "Error: " << std::endl
5378  << "row_or_column_start.size() "
5379  << row_or_column_start.size() << " does not equal "
5380  << "column_or_row_index.size() "
5381  << column_or_row_index.size() << std::endl;
5382  throw OomphLibError(
5383  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
5384  }
5385 
5386  if (value.size() != n_matrix)
5387  {
5388  std::ostringstream error_stream;
5389  error_stream << "Error: " << std::endl
5390  << "value.size() " << value.size() << " does not equal "
5391  << "column_or_row_index.size() "
5392  << column_or_row_index.size() << std::endl
5393  << std::endl
5394  << std::endl;
5395  throw OomphLibError(
5396  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
5397  }
5398 #endif
5399 
5400 
5401  // The idea behind this sparse assembly routine is to use a Vector of
5402  // Vectors of pairs for each complete matrix.
5403  // Each inner Vector stores pairs and holds the row (or column) index
5404  // and the value of the matrix entry.
5405 
5406  // Set up Vector of Vectors to store the entries of each matrix,
5407  // indexed by either the column or row.
5408  Vector<Vector<Vector<std::pair<unsigned, double>>>> matrix_data(n_matrix);
5409 
5410  // Loop over the number of matrices being assembled and resize
5411  // each Vector of Vectors to the number of rows or columns of the matrix
5412  for (unsigned m = 0; m < n_matrix; m++)
5413  {
5414  matrix_data[m].resize(ndof);
5415  }
5416 
5417  // Resize the residuals vectors
5418  for (unsigned v = 0; v < n_vector; v++)
5419  {
5420  residuals[v] = new double[ndof];
5421  for (unsigned i = 0; i < ndof; i++)
5422  {
5423  residuals[v][i] = 0;
5424  }
5425  }
5426 
5427 #ifdef OOMPH_HAS_MPI
5428 
5429  // Storage for assembly time for elements
5430  double t_assemble_start = 0.0;
5431 
5432  // Storage for assembly times
5433  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
5434  {
5435  Elemental_assembly_time.resize(n_elements);
5436  }
5437 
5438 #endif
5439 
5440  //----------------Assemble and populate the vector storage scheme--------
5441  {
5442  // Allocate local storage for the element's contribution to the
5443  // residuals vectors and system matrices of the size of the maximum
5444  // number of dofs in any element
5445  // This means that the storage is only allocated (and deleted) once
5446  Vector<Vector<double>> el_residuals(n_vector);
5447  Vector<DenseMatrix<double>> el_jacobian(n_matrix);
5448 
5449  // Loop over the elements
5450  for (unsigned long e = el_lo; e <= el_hi; e++)
5451  {
5452 #ifdef OOMPH_HAS_MPI
5453  // Time it?
5454  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
5455  {
5456  t_assemble_start = TimingHelpers::timer();
5457  }
5458 #endif
5459 
5460  // Get the pointer to the element
5461  GeneralisedElement* elem_pt = mesh_pt()->element_pt(e);
5462 
5463 #ifdef OOMPH_HAS_MPI
5464  // Ignore halo elements
5465  if (!elem_pt->is_halo())
5466  {
5467 #endif
5468 
5469  // Find number of degrees of freedom in the element
5470  const unsigned nvar = assembly_handler_pt->ndof(elem_pt);
5471 
5472  // Resize the storage for elemental jacobian and residuals
5473  for (unsigned v = 0; v < n_vector; v++)
5474  {
5475  el_residuals[v].resize(nvar);
5476  }
5477  for (unsigned m = 0; m < n_matrix; m++)
5478  {
5479  el_jacobian[m].resize(nvar);
5480  }
5481 
5482  // Now get the residuals and jacobian for the element
5484  elem_pt, el_residuals, el_jacobian);
5485 
5486  //---------------Insert the values into the vectors--------------
5487 
5488  // Loop over the first index of local variables
5489  for (unsigned i = 0; i < nvar; i++)
5490  {
5491  // Get the local equation number
5492  unsigned eqn_number = assembly_handler_pt->eqn_number(elem_pt, i);
5493 
5494  // Add the contribution to the residuals
5495  for (unsigned v = 0; v < n_vector; v++)
5496  {
5497  // Fill in each residuals vector
5498  residuals[v][eqn_number] += el_residuals[v][i];
5499  }
5500 
5501  // Now loop over the other index
5502  for (unsigned j = 0; j < nvar; j++)
5503  {
5504  // Get the number of the unknown
5505  unsigned unknown = assembly_handler_pt->eqn_number(elem_pt, j);
5506 
5507  // Loop over the matrices
5508  // If it's compressed row storage, then our vector of maps
5509  // is indexed by row (equation number)
5510  for (unsigned m = 0; m < n_matrix; m++)
5511  {
5512  // Get the value of the matrix at this point
5513  double value = el_jacobian[m](i, j);
5514  // Only bother to add to the vector if it's non-zero
5515  if (std::fabs(value) > Numerical_zero_for_sparse_assembly)
5516  {
5517  // If it's compressed row storage, then our vector of maps
5518  // is indexed by row (equation number)
5519  if (compressed_row_flag)
5520  {
5521  // Find the correct position and add the data into the
5522  // vectors
5523  const unsigned size = matrix_data[m][eqn_number].size();
5524  for (unsigned k = 0; k <= size; k++)
5525  {
5526  if (k == size)
5527  {
5528  matrix_data[m][eqn_number].push_back(
5529  std::make_pair(unknown, value));
5530  break;
5531  }
5532  else if (matrix_data[m][eqn_number][k].first == unknown)
5533  {
5534  matrix_data[m][eqn_number][k].second += value;
5535  break;
5536  }
5537  }
5538  }
5539  // Otherwise it's compressed column storage and our vector is
5540  // indexed by column (the unknown)
5541  else
5542  {
5543  // Add the data into the vectors in the correct position
5544  const unsigned size = matrix_data[m][unknown].size();
5545  for (unsigned k = 0; k <= size; k++)
5546  {
5547  if (k == size)
5548  {
5549  matrix_data[m][unknown].push_back(
5550  std::make_pair(eqn_number, value));
5551  break;
5552  }
5553  else if (matrix_data[m][unknown][k].first == eqn_number)
5554  {
5555  matrix_data[m][unknown][k].second += value;
5556  break;
5557  }
5558  }
5559  }
5560  }
5561  } // End of loop over matrices
5562  }
5563  }
5564 
5565 #ifdef OOMPH_HAS_MPI
5566  } // endif halo element
5567 #endif
5568 
5569 
5570 #ifdef OOMPH_HAS_MPI
5571  // Time it?
5572  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
5573  {
5575  TimingHelpers::timer() - t_assemble_start;
5576  }
5577 #endif
5578 
5579  } // End of loop over the elements
5580 
5581 
5582  } // End of vector assembly
5583 
5584 
5585 #ifdef OOMPH_HAS_MPI
5586 
5587  // Postprocess timing information and re-allocate distribution of
5588  // elements during subsequent assemblies.
5589  if ((!doing_residuals) && (!Problem_has_been_distributed) &&
5591  {
5593  }
5594 
5595  // We have determined load balancing for current setup.
5596  // This can remain the same until assign_eqn_numbers() is called
5597  // again -- the flag is re-set to true there.
5598  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
5599  {
5601  }
5602 
5603 #endif
5604 
5605 
5606  //-----------Finally we need to convert this vector storage scheme
5607  //------------------------to the containers required by SuperLU
5608 
5609  // Loop over the number of matrices
5610  for (unsigned m = 0; m < n_matrix; m++)
5611  {
5612  // Set the number of rows or columns
5613  row_or_column_start[m] = new int[ndof + 1];
5614 
5615  // fill row_or_column_start and find the number of entries
5616  row_or_column_start[m][0] = 0;
5617  for (unsigned long i = 0; i < ndof; i++)
5618  {
5619  row_or_column_start[m][i + 1] =
5620  row_or_column_start[m][i] + matrix_data[m][i].size();
5621  }
5622  const unsigned entries = row_or_column_start[m][ndof];
5623 
5624  // resize vectors
5625  column_or_row_index[m] = new int[entries];
5626  value[m] = new double[entries];
5627  nnz[m] = entries;
5628 
5629  // Now we merely loop over the number of rows or columns
5630  for (unsigned long i_global = 0; i_global < ndof; i_global++)
5631  {
5632  // If there are no entries in the vector then skip the rest of the loop
5633  if (matrix_data[m][i_global].empty())
5634  {
5635  continue;
5636  }
5637 
5638  // Loop over all the entries in the vectors corresponding to the given
5639  // row or column. It will NOT be ordered
5640  unsigned p = 0;
5641  for (int j = row_or_column_start[m][i_global];
5642  j < row_or_column_start[m][i_global + 1];
5643  j++)
5644  {
5645  column_or_row_index[m][j] = matrix_data[m][i_global][p].first;
5646  value[m][j] = matrix_data[m][i_global][p].second;
5647  ++p;
5648  }
5649  }
5650  } // End of the loop over the matrices
5651 
5653  {
5654  oomph_info << "Pausing at end of sparse assembly." << std::endl;
5655  pause("Check memory usage now.");
5656  }
5657  }
5658 
5659 
5660  //=====================================================================
5661  /// This is a (private) helper function that is used to assemble system
5662  /// matrices in compressed row or column format
5663  /// and compute residual vectors using two vectors.
5664  /// The default action is to assemble the jacobian matrix and
5665  /// residuals for the Newton method. The action can be
5666  /// overloaded at an elemental level by chaging the default
5667  /// behaviour of the function Element::get_all_vectors_and_matrices().
5668  /// column_or_row_index: Column [or row] index of given entry
5669  /// row_or_column_start: Index of first entry for given row [or column]
5670  /// value : Vector of nonzero entries
5671  /// residuals : Residual vector
5672  /// compressed_row_flag: Bool flag to indicate if storage format is
5673  /// compressed row [if false interpretation of
5674  /// arguments is as stated in square brackets].
5675  //=====================================================================
5677  Vector<int*>& column_or_row_index,
5678  Vector<int*>& row_or_column_start,
5679  Vector<double*>& value,
5680  Vector<unsigned>& nnz,
5681  Vector<double*>& residuals,
5682  bool compressed_row_flag)
5683  {
5684  // Total number of elements
5685  const unsigned long n_elements = mesh_pt()->nelement();
5686 
5687  // Default range of elements for distributed problems
5688  unsigned long el_lo = 0;
5689  unsigned long el_hi = n_elements - 1;
5690 
5691 
5692 #ifdef OOMPH_HAS_MPI
5693  // Otherwise just loop over a fraction of the elements
5694  // (This will either have been initialised in
5695  // Problem::set_default_first_and_last_element_for_assembly() or
5696  // will have been re-assigned during a previous assembly loop
5697  // Note that following the re-assignment only the entries
5698  // for the current processor are relevant.
5700  {
5701  el_lo = First_el_for_assembly[Communicator_pt->my_rank()];
5702  el_hi = Last_el_plus_one_for_assembly[Communicator_pt->my_rank()] - 1;
5703  }
5704 #endif
5705 
5706  // number of local eqns
5707  unsigned ndof = this->ndof();
5708 
5709  // Find the number of vectors to be assembled
5710  const unsigned n_vector = residuals.size();
5711 
5712  // Find the number of matrices to be assembled
5713  const unsigned n_matrix = column_or_row_index.size();
5714 
5715  // Locally cache pointer to assembly handler
5717 
5718 #ifdef OOMPH_HAS_MPI
5719  bool doing_residuals = false;
5720  if (dynamic_cast<ParallelResidualsHandler*>(Assembly_handler_pt) != 0)
5721  {
5722  doing_residuals = true;
5723  }
5724 #endif
5725 
5726 // Error check dimensions
5727 #ifdef PARANOID
5728  if (row_or_column_start.size() != n_matrix)
5729  {
5730  std::ostringstream error_stream;
5731  error_stream << "Error: " << std::endl
5732  << "row_or_column_start.size() "
5733  << row_or_column_start.size() << " does not equal "
5734  << "column_or_row_index.size() "
5735  << column_or_row_index.size() << std::endl;
5736  throw OomphLibError(
5737  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
5738  }
5739 
5740  if (value.size() != n_matrix)
5741  {
5742  std::ostringstream error_stream;
5743  error_stream << "Error: " << std::endl
5744  << "value.size() " << value.size() << " does not equal "
5745  << "column_or_row_index.size() "
5746  << column_or_row_index.size() << std::endl
5747  << std::endl
5748  << std::endl;
5749  throw OomphLibError(
5750  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
5751  }
5752 #endif
5753 
5754  // The idea behind this sparse assembly routine is to use Vectors of
5755  // Vectors for the entries in each complete matrix. And a second
5756  // Vector of Vectors stores the global row (or column) indeces. This
5757  // will not have the memory overheads associated with the methods using
5758  // lists or maps, but insertion will be more costly.
5759 
5760  // Set up two vector of vectors to store the entries of each matrix,
5761  // indexed by either the column or row. The entries of the vector for
5762  // each matrix correspond to all the rows or columns of that matrix.
5763  Vector<Vector<Vector<unsigned>>> matrix_row_or_col_indices(n_matrix);
5764  Vector<Vector<Vector<double>>> matrix_values(n_matrix);
5765 
5766  // Loop over the number of matrices being assembled and resize
5767  // each vector of vectors to the number of rows or columns of the matrix
5768  for (unsigned m = 0; m < n_matrix; m++)
5769  {
5770  matrix_row_or_col_indices[m].resize(ndof);
5771  matrix_values[m].resize(ndof);
5772  }
5773 
5774  // Resize the residuals vectors
5775  for (unsigned v = 0; v < n_vector; v++)
5776  {
5777  residuals[v] = new double[ndof];
5778  for (unsigned i = 0; i < ndof; i++)
5779  {
5780  residuals[v][i] = 0;
5781  }
5782  }
5783 
5784 #ifdef OOMPH_HAS_MPI
5785 
5786  // Storage for assembly time for elements
5787  double t_assemble_start = 0.0;
5788 
5789  // Storage for assembly times
5790  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
5791  {
5792  Elemental_assembly_time.resize(n_elements);
5793  }
5794 
5795 #endif
5796 
5797 
5798  //----------------Assemble and populate the vector storage scheme-------
5799  {
5800  // Allocate local storage for the element's contribution to the
5801  // residuals vectors and system matrices of the size of the maximum
5802  // number of dofs in any element
5803  // This means that the storage will only be allocated (and deleted) once
5804  Vector<Vector<double>> el_residuals(n_vector);
5805  Vector<DenseMatrix<double>> el_jacobian(n_matrix);
5806 
5807  // Loop over the elements
5808  for (unsigned long e = el_lo; e <= el_hi; e++)
5809  {
5810 #ifdef OOMPH_HAS_MPI
5811  // Time it?
5812  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
5813  {
5814  t_assemble_start = TimingHelpers::timer();
5815  }
5816 #endif
5817 
5818  // Get the pointer to the element
5819  GeneralisedElement* elem_pt = mesh_pt()->element_pt(e);
5820 
5821 #ifdef OOMPH_HAS_MPI
5822  // Ignore halo elements
5823  if (!elem_pt->is_halo())
5824  {
5825 #endif
5826 
5827  // Find number of degrees of freedom in the element
5828  const unsigned nvar = assembly_handler_pt->ndof(elem_pt);
5829 
5830  // Resize the storage for elemental jacobian and residuals
5831  for (unsigned v = 0; v < n_vector; v++)
5832  {
5833  el_residuals[v].resize(nvar);
5834  }
5835  for (unsigned m = 0; m < n_matrix; m++)
5836  {
5837  el_jacobian[m].resize(nvar);
5838  }
5839 
5840  // Now get the residuals and jacobian for the element
5842  elem_pt, el_residuals, el_jacobian);
5843 
5844  //---------------Insert the values into the vectors--------------
5845 
5846  // Loop over the first index of local variables
5847  for (unsigned i = 0; i < nvar; i++)
5848  {
5849  // Get the local equation number
5850  unsigned eqn_number = assembly_handler_pt->eqn_number(elem_pt, i);
5851 
5852  // Add the contribution to the residuals
5853  for (unsigned v = 0; v < n_vector; v++)
5854  {
5855  // Fill in each residuals vector
5856  residuals[v][eqn_number] += el_residuals[v][i];
5857  }
5858 
5859  // Now loop over the other index
5860  for (unsigned j = 0; j < nvar; j++)
5861  {
5862  // Get the number of the unknown
5863  unsigned unknown = assembly_handler_pt->eqn_number(elem_pt, j);
5864 
5865  // Loop over the matrices
5866  // If it's compressed row storage, then our vector of maps
5867  // is indexed by row (equation number)
5868  for (unsigned m = 0; m < n_matrix; m++)
5869  {
5870  // Get the value of the matrix at this point
5871  double value = el_jacobian[m](i, j);
5872  // Only bother to add to the vector if it's non-zero
5873  if (std::fabs(value) > Numerical_zero_for_sparse_assembly)
5874  {
5875  // If it's compressed row storage, then our vector of maps
5876  // is indexed by row (equation number)
5877  if (compressed_row_flag)
5878  {
5879  // Find the correct position and add the data into the
5880  // vectors
5881  const unsigned size =
5882  matrix_row_or_col_indices[m][eqn_number].size();
5883 
5884  for (unsigned k = 0; k <= size; k++)
5885  {
5886  if (k == size)
5887  {
5888  matrix_row_or_col_indices[m][eqn_number].push_back(
5889  unknown);
5890  matrix_values[m][eqn_number].push_back(value);
5891  break;
5892  }
5893  else if (matrix_row_or_col_indices[m][eqn_number][k] ==
5894  unknown)
5895  {
5896  matrix_values[m][eqn_number][k] += value;
5897  break;
5898  }
5899  }
5900  }
5901  // Otherwise it's compressed column storage and our vector is
5902  // indexed by column (the unknown)
5903  else
5904  {
5905  // Add the data into the vectors in the correct position
5906  const unsigned size =
5907  matrix_row_or_col_indices[m][unknown].size();
5908  for (unsigned k = 0; k <= size; k++)
5909  {
5910  if (k == size)
5911  {
5912  matrix_row_or_col_indices[m][unknown].push_back(
5913  eqn_number);
5914  matrix_values[m][unknown].push_back(value);
5915  break;
5916  }
5917  else if (matrix_row_or_col_indices[m][unknown][k] ==
5918  eqn_number)
5919  {
5920  matrix_values[m][unknown][k] += value;
5921  break;
5922  }
5923  }
5924  }
5925  }
5926  } // End of loop over matrices
5927  }
5928  }
5929 
5930 #ifdef OOMPH_HAS_MPI
5931  } // endif halo element
5932 #endif
5933 
5934 
5935 #ifdef OOMPH_HAS_MPI
5936  // Time it?
5937  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
5938  {
5940  TimingHelpers::timer() - t_assemble_start;
5941  }
5942 #endif
5943 
5944  } // End of loop over the elements
5945 
5946  } // End of vector assembly
5947 
5948 
5949 #ifdef OOMPH_HAS_MPI
5950 
5951  // Postprocess timing information and re-allocate distribution of
5952  // elements during subsequent assemblies.
5953  if ((!doing_residuals) && (!Problem_has_been_distributed) &&
5955  {
5957  }
5958 
5959  // We have determined load balancing for current setup.
5960  // This can remain the same until assign_eqn_numbers() is called
5961  // again -- the flag is re-set to true there.
5962  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
5963  {
5965  }
5966 
5967 #endif
5968 
5969  //-----------Finally we need to convert this lousy vector storage scheme
5970  //------------------------to the containers required by SuperLU
5971 
5972  // Loop over the number of matrices
5973  for (unsigned m = 0; m < n_matrix; m++)
5974  {
5975  // Set the number of rows or columns
5976  row_or_column_start[m] = new int[ndof + 1];
5977 
5978  // fill row_or_column_start and find the number of entries
5979  row_or_column_start[m][0] = 0;
5980  for (unsigned long i = 0; i < ndof; i++)
5981  {
5982  row_or_column_start[m][i + 1] =
5983  row_or_column_start[m][i] + matrix_values[m][i].size();
5984  }
5985  const unsigned entries = row_or_column_start[m][ndof];
5986 
5987  // resize vectors
5988  column_or_row_index[m] = new int[entries];
5989  value[m] = new double[entries];
5990  nnz[m] = entries;
5991 
5992  // Now we merely loop over the number of rows or columns
5993  for (unsigned long i_global = 0; i_global < ndof; i_global++)
5994  {
5995  // If there are no entries in the vector then skip the rest of the loop
5996  if (matrix_values[m][i_global].empty())
5997  {
5998  continue;
5999  }
6000 
6001  // Loop over all the entries in the vectors corresponding to the given
6002  // row or column. It will NOT be ordered
6003  unsigned p = 0;
6004  for (int j = row_or_column_start[m][i_global];
6005  j < row_or_column_start[m][i_global + 1];
6006  j++)
6007  {
6008  column_or_row_index[m][j] = matrix_row_or_col_indices[m][i_global][p];
6009  value[m][j] = matrix_values[m][i_global][p];
6010  ++p;
6011  }
6012  }
6013  } // End of the loop over the matrices
6014 
6016  {
6017  oomph_info << "Pausing at end of sparse assembly." << std::endl;
6018  pause("Check memory usage now.");
6019  }
6020  }
6021 
6022 
6023  //=====================================================================
6024  /// This is a (private) helper function that is used to assemble system
6025  /// matrices in compressed row or column format
6026  /// and compute residual vectors using two vectors.
6027  /// The default action is to assemble the jacobian matrix and
6028  /// residuals for the Newton method. The action can be
6029  /// overloaded at an elemental level by chaging the default
6030  /// behaviour of the function Element::get_all_vectors_and_matrices().
6031  /// column_or_row_index: Column [or row] index of given entry
6032  /// row_or_column_start: Index of first entry for given row [or column]
6033  /// value : Vector of nonzero entries
6034  /// residuals : Residual vector
6035  /// compressed_row_flag: Bool flag to indicate if storage format is
6036  /// compressed row [if false interpretation of
6037  /// arguments is as stated in square brackets].
6038  //=====================================================================
6040  Vector<int*>& column_or_row_index,
6041  Vector<int*>& row_or_column_start,
6042  Vector<double*>& value,
6043  Vector<unsigned>& nnz,
6044  Vector<double*>& residuals,
6045  bool compressed_row_flag)
6046  {
6047  // Total number of elements
6048  const unsigned long n_elements = mesh_pt()->nelement();
6049 
6050  // Default range of elements for distributed problems
6051  unsigned long el_lo = 0;
6052  unsigned long el_hi = n_elements - 1;
6053 
6054 
6055 #ifdef OOMPH_HAS_MPI
6056  // Otherwise just loop over a fraction of the elements
6057  // (This will either have been initialised in
6058  // Problem::set_default_first_and_last_element_for_assembly() or
6059  // will have been re-assigned during a previous assembly loop
6060  // Note that following the re-assignment only the entries
6061  // for the current processor are relevant.
6063  {
6064  el_lo = First_el_for_assembly[Communicator_pt->my_rank()];
6065  el_hi = Last_el_plus_one_for_assembly[Communicator_pt->my_rank()] - 1;
6066  }
6067 #endif
6068 
6069  // number of local eqns
6070  unsigned ndof = this->ndof();
6071 
6072  // Find the number of vectors to be assembled
6073  const unsigned n_vector = residuals.size();
6074 
6075  // Find the number of matrices to be assembled
6076  const unsigned n_matrix = column_or_row_index.size();
6077 
6078  // Locally cache pointer to assembly handler
6080 
6081 #ifdef OOMPH_HAS_MPI
6082  bool doing_residuals = false;
6083  if (dynamic_cast<ParallelResidualsHandler*>(Assembly_handler_pt) != 0)
6084  {
6085  doing_residuals = true;
6086  }
6087 #endif
6088 
6089 // Error check dimensions
6090 #ifdef PARANOID
6091  if (row_or_column_start.size() != n_matrix)
6092  {
6093  std::ostringstream error_stream;
6094  error_stream << "Error: " << std::endl
6095  << "row_or_column_start.size() "
6096  << row_or_column_start.size() << " does not equal "
6097  << "column_or_row_index.size() "
6098  << column_or_row_index.size() << std::endl;
6099  throw OomphLibError(
6100  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
6101  }
6102 
6103  if (value.size() != n_matrix)
6104  {
6105  std::ostringstream error_stream;
6106  error_stream << "Error: " << std::endl
6107  << "value.size() " << value.size() << " does not equal "
6108  << "column_or_row_index.size() "
6109  << column_or_row_index.size() << std::endl
6110  << std::endl
6111  << std::endl;
6112  throw OomphLibError(
6113  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
6114  }
6115 #endif
6116 
6117  // The idea behind this sparse assembly routine is to use Vectors of
6118  // Vectors for the entries in each complete matrix. And a second
6119  // Vector of Vectors stores the global row (or column) indeces. This
6120  // will not have the memory overheads associated with the methods using
6121  // lists or maps, but insertion will be more costly.
6122 
6123  // Set up two vector of vectors to store the entries of each matrix,
6124  // indexed by either the column or row. The entries of the vector for
6125  // each matrix correspond to all the rows or columns of that matrix.
6126  Vector<unsigned**> matrix_row_or_col_indices(n_matrix);
6127  Vector<double**> matrix_values(n_matrix);
6128 
6129  // Loop over the number of matrices being assembled and resize
6130  // each vector of vectors to the number of rows or columns of the matrix
6131  for (unsigned m = 0; m < n_matrix; m++)
6132  {
6133  matrix_row_or_col_indices[m] = new unsigned*[ndof];
6134  matrix_values[m] = new double*[ndof];
6135  }
6136 
6137  // Resize the residuals vectors
6138  for (unsigned v = 0; v < n_vector; v++)
6139  {
6140  residuals[v] = new double[ndof];
6141  for (unsigned i = 0; i < ndof; i++)
6142  {
6143  residuals[v][i] = 0;
6144  }
6145  }
6146 
6147 #ifdef OOMPH_HAS_MPI
6148 
6149  // Storage for assembly time for elements
6150  double t_assemble_start = 0.0;
6151 
6152  // Storage for assembly times
6153  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
6154  {
6155  Elemental_assembly_time.resize(n_elements);
6156  }
6157 
6158 #endif
6159 
6160  // number of coefficients in each row
6161  Vector<Vector<unsigned>> ncoef(n_matrix);
6162  for (unsigned m = 0; m < n_matrix; m++)
6163  {
6164  ncoef[m].resize(ndof, 0);
6165  }
6166 
6168  {
6170  for (unsigned m = 0; m < n_matrix; m++)
6171  {
6173  }
6174  }
6175 
6176  //----------------Assemble and populate the vector storage scheme-------
6177  {
6178  // Allocate local storage for the element's contribution to the
6179  // residuals vectors and system matrices of the size of the maximum
6180  // number of dofs in any element
6181  // This means that the storage will only be allocated (and deleted) once
6182  Vector<Vector<double>> el_residuals(n_vector);
6183  Vector<DenseMatrix<double>> el_jacobian(n_matrix);
6184 
6185  // Loop over the elements
6186  for (unsigned long e = el_lo; e <= el_hi; e++)
6187  {
6188 #ifdef OOMPH_HAS_MPI
6189  // Time it?
6190  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
6191  {
6192  t_assemble_start = TimingHelpers::timer();
6193  }
6194 #endif
6195 
6196  // Get the pointer to the element
6197  GeneralisedElement* elem_pt = mesh_pt()->element_pt(e);
6198 
6199 #ifdef OOMPH_HAS_MPI
6200  // Ignore halo elements
6201  if (!elem_pt->is_halo())
6202  {
6203 #endif
6204 
6205  // Find number of degrees of freedom in the element
6206  const unsigned nvar = assembly_handler_pt->ndof(elem_pt);
6207 
6208  // Resize the storage for elemental jacobian and residuals
6209  for (unsigned v = 0; v < n_vector; v++)
6210  {
6211  el_residuals[v].resize(nvar);
6212  }
6213  for (unsigned m = 0; m < n_matrix; m++)
6214  {
6215  el_jacobian[m].resize(nvar);
6216  }
6217 
6218  // Now get the residuals and jacobian for the element
6220  elem_pt, el_residuals, el_jacobian);
6221 
6222  //---------------Insert the values into the vectors--------------
6223 
6224  // Loop over the first index of local variables
6225  for (unsigned i = 0; i < nvar; i++)
6226  {
6227  // Get the local equation number
6228  unsigned eqn_number = assembly_handler_pt->eqn_number(elem_pt, i);
6229 
6230  // Add the contribution to the residuals
6231  for (unsigned v = 0; v < n_vector; v++)
6232  {
6233  // Fill in each residuals vector
6234  residuals[v][eqn_number] += el_residuals[v][i];
6235  }
6236 
6237  // Now loop over the other index
6238  for (unsigned j = 0; j < nvar; j++)
6239  {
6240  // Get the number of the unknown
6241  unsigned unknown = assembly_handler_pt->eqn_number(elem_pt, j);
6242 
6243  // Loop over the matrices
6244  // If it's compressed row storage, then our vector of maps
6245  // is indexed by row (equation number)
6246  for (unsigned m = 0; m < n_matrix; m++)
6247  {
6248  // Get the value of the matrix at this point
6249  double value = el_jacobian[m](i, j);
6250  // Only bother to add to the vector if it's non-zero
6251  if (std::fabs(value) > Numerical_zero_for_sparse_assembly)
6252  {
6253  // number of entrys in this row
6254  const unsigned size = ncoef[m][eqn_number];
6255 
6256  // if no data has been allocated for this row then allocate
6257  if (size == 0)
6258  {
6259  // do we have previous allocation data
6261  [m][eqn_number] != 0)
6262  {
6263  matrix_row_or_col_indices[m][eqn_number] = new unsigned
6265  [m][eqn_number]];
6266  matrix_values[m][eqn_number] = new double
6268  [m][eqn_number]];
6269  }
6270  else
6271  {
6272  matrix_row_or_col_indices[m][eqn_number] = new unsigned
6274  matrix_values[m][eqn_number] = new double
6277  [m][eqn_number] =
6279  }
6280  }
6281 
6282  // If it's compressed row storage, then our vector of maps
6283  // is indexed by row (equation number)
6284  if (compressed_row_flag)
6285  {
6286  // next add the data
6287  for (unsigned k = 0; k <= size; k++)
6288  {
6289  if (k == size)
6290  {
6291  // do we need to allocate more storage
6293  [m][eqn_number] == ncoef[m][eqn_number])
6294  {
6295  unsigned new_allocation =
6296  ncoef[m][eqn_number] +
6298  double* new_values = new double[new_allocation];
6299  unsigned* new_indices = new unsigned[new_allocation];
6300  for (unsigned c = 0; c < ncoef[m][eqn_number]; c++)
6301  {
6302  new_values[c] = matrix_values[m][eqn_number][c];
6303  new_indices[c] =
6304  matrix_row_or_col_indices[m][eqn_number][c];
6305  }
6306  delete[] matrix_values[m][eqn_number];
6307  delete[] matrix_row_or_col_indices[m][eqn_number];
6308  matrix_values[m][eqn_number] = new_values;
6309  matrix_row_or_col_indices[m][eqn_number] =
6310  new_indices;
6312  [m][eqn_number] = new_allocation;
6313  }
6314  // and now add the data
6315  unsigned entry = ncoef[m][eqn_number];
6316  ncoef[m][eqn_number]++;
6317  matrix_row_or_col_indices[m][eqn_number][entry] =
6318  unknown;
6319  matrix_values[m][eqn_number][entry] = value;
6320  break;
6321  }
6322  else if (matrix_row_or_col_indices[m][eqn_number][k] ==
6323  unknown)
6324  {
6325  matrix_values[m][eqn_number][k] += value;
6326  break;
6327  }
6328  }
6329  }
6330  // Otherwise it's compressed column storage and our vector is
6331  // indexed by column (the unknown)
6332  else
6333  {
6334  // Add the data into the vectors in the correct position
6335  for (unsigned k = 0; k <= size; k++)
6336  {
6337  if (k == size)
6338  {
6339  // do we need to allocate more storage
6341  [m][unknown] == ncoef[m][unknown])
6342  {
6343  unsigned new_allocation =
6344  ncoef[m][unknown] +
6346  double* new_values = new double[new_allocation];
6347  unsigned* new_indices = new unsigned[new_allocation];
6348  for (unsigned c = 0; c < ncoef[m][unknown]; c++)
6349  {
6350  new_values[c] = matrix_values[m][unknown][c];
6351  new_indices[c] =
6352  matrix_row_or_col_indices[m][unknown][c];
6353  }
6354  delete[] matrix_values[m][unknown];
6355  delete[] matrix_row_or_col_indices[m][unknown];
6357  [m][unknown] = new_allocation;
6358  }
6359  // and now add the data
6360  unsigned entry = ncoef[m][unknown];
6361  ncoef[m][unknown]++;
6362  matrix_row_or_col_indices[m][unknown][entry] =
6363  eqn_number;
6364  matrix_values[m][unknown][entry] = value;
6365  break;
6366  }
6367  else if (matrix_row_or_col_indices[m][unknown][k] ==
6368  eqn_number)
6369  {
6370  matrix_values[m][unknown][k] += value;
6371  break;
6372  }
6373  }
6374  }
6375  }
6376  } // End of loop over matrices
6377  }
6378  }
6379 
6380 #ifdef OOMPH_HAS_MPI
6381  } // endif halo element
6382 #endif
6383 
6384 
6385 #ifdef OOMPH_HAS_MPI
6386  // Time it?
6387  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
6388  {
6390  TimingHelpers::timer() - t_assemble_start;
6391  }
6392 #endif
6393 
6394  } // End of loop over the elements
6395 
6396  } // End of vector assembly
6397 
6398 
6399 #ifdef OOMPH_HAS_MPI
6400 
6401  // Postprocess timing information and re-allocate distribution of
6402  // elements during subsequent assemblies.
6403  if ((!doing_residuals) && (!Problem_has_been_distributed) &&
6405  {
6407  }
6408 
6409  // We have determined load balancing for current setup.
6410  // This can remain the same until assign_eqn_numbers() is called
6411  // again -- the flag is re-set to true there.
6412  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
6413  {
6415  }
6416 
6417 #endif
6418 
6419  //-----------Finally we need to convert this lousy vector storage scheme
6420  //------------------------to the containers required by SuperLU
6421 
6422  // Loop over the number of matrices
6423  for (unsigned m = 0; m < n_matrix; m++)
6424  {
6425  // Set the number of rows or columns
6426  row_or_column_start[m] = new int[ndof + 1];
6427 
6428  // fill row_or_column_start and find the number of entries
6429  row_or_column_start[m][0] = 0;
6430  for (unsigned long i = 0; i < ndof; i++)
6431  {
6432  row_or_column_start[m][i + 1] = row_or_column_start[m][i] + ncoef[m][i];
6434  }
6435  const unsigned entries = row_or_column_start[m][ndof];
6436 
6437  // resize vectors
6438  column_or_row_index[m] = new int[entries];
6439  value[m] = new double[entries];
6440  nnz[m] = entries;
6441 
6442  // Now we merely loop over the number of rows or columns
6443  for (unsigned long i_global = 0; i_global < ndof; i_global++)
6444  {
6445  // If there are no entries in the vector then skip the rest of the loop
6446  if (ncoef[m][i_global] == 0)
6447  {
6448  continue;
6449  }
6450 
6451  // Loop over all the entries in the vectors corresponding to the given
6452  // row or column. It will NOT be ordered
6453  unsigned p = 0;
6454  for (int j = row_or_column_start[m][i_global];
6455  j < row_or_column_start[m][i_global + 1];
6456  j++)
6457  {
6458  column_or_row_index[m][j] = matrix_row_or_col_indices[m][i_global][p];
6459  value[m][j] = matrix_values[m][i_global][p];
6460  ++p;
6461  }
6462 
6463  // and delete
6464  delete[] matrix_row_or_col_indices[m][i_global];
6465  delete[] matrix_values[m][i_global];
6466  }
6467 
6468  //
6469  delete[] matrix_row_or_col_indices[m];
6470  delete[] matrix_values[m];
6471  } // End of the loop over the matrices
6472 
6474  {
6475  oomph_info << "Pausing at end of sparse assembly." << std::endl;
6476  pause("Check memory usage now.");
6477  }
6478  }
6479 
6480 
6481 #ifdef OOMPH_HAS_MPI
6482  //=======================================================================
6483  /// Helper method that returns the global equations to which
6484  /// the elements in the range el_lo to el_hi contribute on this
6485  /// processor
6486  //=======================================================================
6487  void Problem::get_my_eqns(AssemblyHandler* const& assembly_handler_pt,
6488  const unsigned& el_lo,
6489  const unsigned& el_hi,
6490  Vector<unsigned>& my_eqns)
6491  {
6492  // Index to keep track of the equations counted
6493  unsigned my_eqns_index = 0;
6494 
6495  // Loop over the selection of elements
6496  for (unsigned long e = el_lo; e <= el_hi; e++)
6497  {
6498  // Get the pointer to the element
6499  GeneralisedElement* elem_pt = this->mesh_pt()->element_pt(e);
6500 
6501  // Ignore halo elements
6502  if (!elem_pt->is_halo())
6503  {
6504  // Find number of degrees of freedom in the element
6505  const unsigned nvar = assembly_handler_pt->ndof(elem_pt);
6506  // Add the number of dofs to the current size of my_eqns
6507  my_eqns.resize(my_eqns_index + nvar);
6508 
6509  // Loop over the first index of local variables
6510  for (unsigned i = 0; i < nvar; i++)
6511  {
6512  // Get the local equation number
6513  unsigned global_eqn_number =
6514  assembly_handler_pt->eqn_number(elem_pt, i);
6515  // Add into the vector
6516  my_eqns[my_eqns_index + i] = global_eqn_number;
6517  }
6518  // Update the number of elements in the vector
6519  my_eqns_index += nvar;
6520  }
6521  }
6522 
6523  // now sort and remove duplicate entries in the vector
6524  std::sort(my_eqns.begin(), my_eqns.end());
6525  Vector<unsigned>::iterator it = std::unique(my_eqns.begin(), my_eqns.end());
6526  my_eqns.resize(it - my_eqns.begin());
6527  }
6528 
6529 
6530  //=============================================================================
6531  /// Helper method to assemble CRDoubleMatrices from distributed
6532  /// on multiple processors.
6533  //=============================================================================
6535  const LinearAlgebraDistribution* const& target_dist_pt,
6536  Vector<int*>& column_indices,
6537  Vector<int*>& row_start,
6538  Vector<double*>& values,
6539  Vector<unsigned>& nnz,
6540  Vector<double*>& residuals)
6541  {
6542  // Time assembly
6543  double t_start = TimingHelpers::timer();
6544 
6545  // my rank and nproc
6546  unsigned my_rank = Communicator_pt->my_rank();
6547  unsigned nproc = Communicator_pt->nproc();
6548 
6549  // Total number of elements
6550  const unsigned long n_elements = mesh_pt()->nelement();
6551 
6552 #ifdef PARANOID
6553  // No elements? This is usually a sign that the problem distribution has
6554  // led to one processor not having any elements. Either
6555  // a sign of something having gone wrong or a relatively small
6556  // problem on a huge number of processors
6557  if (n_elements == 0)
6558  {
6559  std::ostringstream error_stream;
6560  error_stream << "Processsor " << my_rank << " has no elements. \n"
6561  << "This is usually a sign that the problem distribution \n"
6562  << "or the load balancing have gone wrong.";
6563  OomphLibWarning(error_stream.str(),
6564  "Problem::parallel_sparse_assemble()",
6565  OOMPH_EXCEPTION_LOCATION);
6566  }
6567 #endif
6568 
6569 
6570  // Default range of elements for distributed problems.
6571  unsigned long el_lo = 0;
6572  unsigned long el_hi_plus_one = n_elements;
6573 
6574  // Otherwise just loop over a fraction of the elements
6575  // (This will either have been initialised in
6576  // Problem::set_default_first_and_last_element_for_assembly() or
6577  // will have been re-assigned during a previous assembly loop
6578  // Note that following the re-assignment only the entries
6579  // for the current processor are relevant.
6581  {
6582  el_lo = First_el_for_assembly[my_rank];
6583  el_hi_plus_one = Last_el_plus_one_for_assembly[my_rank];
6584  }
6585 
6586  // Find the number of vectors to be assembled
6587  const unsigned n_vector = residuals.size();
6588 
6589  // Find the number of matrices to be assembled
6590  const unsigned n_matrix = column_indices.size();
6591 
6592  // Locally cache pointer to assembly handler
6594 
6595  bool doing_residuals = false;
6596  if (dynamic_cast<ParallelResidualsHandler*>(Assembly_handler_pt) != 0)
6597  {
6598  doing_residuals = true;
6599  }
6600 
6601 // Error check dimensions
6602 #ifdef PARANOID
6603  if (row_start.size() != n_matrix)
6604  {
6605  std::ostringstream error_stream;
6606  error_stream << "Error: " << std::endl
6607  << "row_or_column_start.size() " << row_start.size()
6608  << " does not equal "
6609  << "column_or_row_index.size() " << column_indices.size()
6610  << std::endl;
6611  throw OomphLibError(
6612  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
6613  }
6614 
6615  if (values.size() != n_matrix)
6616  {
6617  std::ostringstream error_stream;
6618  error_stream << "Error: " << std::endl
6619  << "value.size() " << values.size() << " does not equal "
6620  << "column_or_row_index.size() " << column_indices.size()
6621  << std::endl
6622  << std::endl
6623  << std::endl;
6624  throw OomphLibError(
6625  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
6626  }
6627 #endif
6628 
6629 
6630  // start by assembling the sorted set of equations to which this processor
6631  // contributes. Essentially this is every global equation that features in
6632  // all the non-halo elements. This may not be the same as the locally-stored
6633  // dofs because some of the Nodes in non-halo elements may actually
6634  // be halos.
6635  //======================================================================
6636  Vector<unsigned> my_eqns;
6637  if (n_elements != 0)
6638  {
6639  this->get_my_eqns(
6640  assembly_handler_pt, el_lo, el_hi_plus_one - 1, my_eqns);
6641  }
6642 
6643  // number of equations
6644  unsigned my_n_eqn = my_eqns.size();
6645 
6646  // next we assemble the data into an array of arrays
6647  // =================================================
6648  // The idea behind this sparse assembly routine is to use an array of
6649  // arrays for the entries in each complete matrix. And a second
6650  // array of arrays stores the global row (or column) indeces.
6651 
6652  // Set up two vector of vectors to store the entries of each matrix,
6653  // indexed by either the column or row. The entries of the vector for
6654  // each matrix correspond to all the rows or columns of that matrix.
6655  Vector<unsigned**> matrix_col_indices(n_matrix);
6656  Vector<double**> matrix_values(n_matrix);
6657 
6658  // Loop over the number of matrices being assembled and resize
6659  // each vector of vectors to the number of rows or columns of the matrix
6660  for (unsigned m = 0; m < n_matrix; m++)
6661  {
6662  matrix_col_indices[m] = new unsigned*[my_n_eqn];
6663  matrix_values[m] = new double*[my_n_eqn];
6664  for (unsigned i = 0; i < my_n_eqn; i++)
6665  {
6666  matrix_col_indices[m][i] = 0;
6667  matrix_values[m][i] = 0;
6668  }
6669  }
6670 
6671  // Resize the residuals vectors
6672  Vector<double*> residuals_data(n_vector);
6673  for (unsigned v = 0; v < n_vector; v++)
6674  {
6675  residuals_data[v] = new double[my_n_eqn];
6676  for (unsigned i = 0; i < my_n_eqn; i++)
6677  {
6678  residuals_data[v][i] = 0;
6679  }
6680  }
6681 
6682  // Storage for assembly time for elements
6683  double t_assemble_start = 0.0;
6684 
6685  // Storage for assembly times
6686  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
6687  {
6688  Elemental_assembly_time.resize(n_elements);
6689  }
6690 
6691  // number of coefficients in each row
6692  Vector<Vector<unsigned>> ncoef(n_matrix);
6693  for (unsigned m = 0; m < n_matrix; m++)
6694  {
6695  ncoef[m].resize(my_n_eqn, 0);
6696  }
6697 
6698  // Sparse_assemble_with_arrays_previous_allocation stores the number of
6699  // coefs in each row.
6700  // if a matrix of this size has not been assembled before then resize this
6701  // storage
6703  {
6705  for (unsigned m = 0; m < n_matrix; m++)
6706  {
6707  Sparse_assemble_with_arrays_previous_allocation[m].resize(my_n_eqn, 0);
6708  }
6709  }
6710 
6711 
6712  // assemble and populate an array based storage scheme
6713  {
6714  // Allocate local storage for the element's contribution to the
6715  // residuals vectors and system matrices of the size of the maximum
6716  // number of dofs in any element
6717  // This means that the storage will only be allocated (and deleted) once
6718  Vector<Vector<double>> el_residuals(n_vector);
6719  Vector<DenseMatrix<double>> el_jacobian(n_matrix);
6720 
6721  // Loop over the elements
6722  for (unsigned long e = el_lo; e < el_hi_plus_one; e++)
6723  {
6724  // Time it?
6725  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
6726  {
6727  t_assemble_start = TimingHelpers::timer();
6728  }
6729 
6730  // Get the pointer to the element
6731  GeneralisedElement* elem_pt = mesh_pt()->element_pt(e);
6732 
6733  // Ignore halo elements
6734  if (!elem_pt->is_halo())
6735  {
6736  // Find number of degrees of freedom in the element
6737  const unsigned nvar = assembly_handler_pt->ndof(elem_pt);
6738 
6739  // Resize the storage for elemental jacobian and residuals
6740  for (unsigned v = 0; v < n_vector; v++)
6741  {
6742  el_residuals[v].resize(nvar);
6743  }
6744  for (unsigned m = 0; m < n_matrix; m++)
6745  {
6746  el_jacobian[m].resize(nvar);
6747  }
6748 
6749  // Now get the residuals and jacobian for the element
6751  elem_pt, el_residuals, el_jacobian);
6752 
6753  //---------------Insert the values into the vectors--------------
6754 
6755  // Loop over the first index of local variables
6756  for (unsigned i = 0; i < nvar; i++)
6757  {
6758  // Get the local equation number
6759  unsigned global_eqn_number =
6760  assembly_handler_pt->eqn_number(elem_pt, i);
6761 
6762  // determine the element number in my set of eqns using the
6763  // bisection method
6764  int left = 0;
6765  int right = my_n_eqn - 1;
6766  int eqn_number = right / 2;
6767  while (my_eqns[eqn_number] != global_eqn_number)
6768  {
6769  if (left == right)
6770  {
6771  // Check that the residuals associated with the
6772  // eqn number that can't be found are all zero
6773  bool broken = false;
6774  for (unsigned v = 0; v < n_vector; v++)
6775  {
6776  if (el_residuals[v][i] != 0.0)
6777  {
6778  broken = true;
6779  break;
6780  }
6781  }
6782 
6783  // Now loop over the other index to check the entries
6784  // in the appropriate row of the Jacobians are zero too
6785  for (unsigned j = 0; j < nvar; j++)
6786  {
6787  // Get the number of the unknown
6788  // unsigned unknown =
6789  // assembly_handler_pt->eqn_number(elem_pt,j);
6790 
6791  // Loop over the matrices
6792  // If it's compressed row storage, then our vector of maps
6793  // is indexed by row (equation number)
6794  for (unsigned m = 0; m < n_matrix; m++)
6795  {
6796  // Get the value of the matrix at this point
6797  double value = el_jacobian[m](i, j);
6798  if (value != 0.0)
6799  {
6800  broken = true;
6801  break;
6802  }
6803  if (broken) break;
6804  }
6805  }
6806 
6807  if (broken)
6808  {
6809  std::ostringstream error_stream;
6810  error_stream
6811  << "Internal Error: " << std::endl
6812  << "Could not find global equation number "
6813  << global_eqn_number
6814  << " in my_eqns vector of equation numbers but\n"
6815  << "at least one entry in the residual vector is nonzero.";
6816  throw OomphLibError(error_stream.str(),
6817  OOMPH_CURRENT_FUNCTION,
6818  OOMPH_EXCEPTION_LOCATION);
6819  }
6820  else
6821  {
6822  break;
6823  }
6824  }
6825  if (my_eqns[eqn_number] > global_eqn_number)
6826  {
6827  right = std::max(eqn_number - 1, left);
6828  }
6829  else
6830  {
6831  left = std::min(eqn_number + 1, right);
6832  }
6833  eqn_number = (right + left) / 2;
6834  }
6835 
6836  // Add the contribution to the residuals
6837  for (unsigned v = 0; v < n_vector; v++)
6838  {
6839  // Fill in each residuals vector
6840  residuals_data[v][eqn_number] += el_residuals[v][i];
6841  }
6842 
6843  // Now loop over the other index
6844  for (unsigned j = 0; j < nvar; j++)
6845  {
6846  // Get the number of the unknown
6847  unsigned unknown = assembly_handler_pt->eqn_number(elem_pt, j);
6848 
6849  // Loop over the matrices
6850  // If it's compressed row storage, then our vector of maps
6851  // is indexed by row (equation number)
6852  for (unsigned m = 0; m < n_matrix; m++)
6853  {
6854  // Get the value of the matrix at this point
6855  double value = el_jacobian[m](i, j);
6856  // Only bother to add to the vector if it's non-zero
6857  if (std::fabs(value) > Numerical_zero_for_sparse_assembly)
6858  {
6859  // number of entrys in this row
6860  const unsigned size = ncoef[m][eqn_number];
6861 
6862  // if no data has been allocated for this row then allocate
6863  if (size == 0)
6864  {
6865  // do we have previous allocation data
6867  [m][eqn_number] != 0)
6868  {
6869  matrix_col_indices[m][eqn_number] = new unsigned
6871  [m][eqn_number]];
6872 
6873  matrix_values[m][eqn_number] = new double
6875  [m][eqn_number]];
6876  }
6877  else
6878  {
6879  matrix_col_indices[m][eqn_number] = new unsigned
6881 
6882  matrix_values[m][eqn_number] = new double
6884 
6886  [m][eqn_number] =
6888  }
6889  }
6890 
6891  // next add the data
6892  for (unsigned k = 0; k <= size; k++)
6893  {
6894  if (k == size)
6895  {
6896  // do we need to allocate more storage
6898  [m][eqn_number] == ncoef[m][eqn_number])
6899  {
6900  unsigned new_allocation =
6901  ncoef[m][eqn_number] +
6903  double* new_values = new double[new_allocation];
6904  unsigned* new_indices = new unsigned[new_allocation];
6905  for (unsigned c = 0; c < ncoef[m][eqn_number]; c++)
6906  {
6907  new_values[c] = matrix_values[m][eqn_number][c];
6908  new_indices[c] = matrix_col_indices[m][eqn_number][c];
6909  }
6910  delete[] matrix_values[m][eqn_number];
6911  delete[] matrix_col_indices[m][eqn_number];
6912 
6913  matrix_values[m][eqn_number] = new_values;
6914  matrix_col_indices[m][eqn_number] = new_indices;
6915 
6917  [m][eqn_number] = new_allocation;
6918  }
6919  // and now add the data
6920  unsigned entry = ncoef[m][eqn_number];
6921  ncoef[m][eqn_number]++;
6922  matrix_col_indices[m][eqn_number][entry] = unknown;
6923  matrix_values[m][eqn_number][entry] = value;
6924  break;
6925  }
6926  else if (matrix_col_indices[m][eqn_number][k] == unknown)
6927  {
6928  matrix_values[m][eqn_number][k] += value;
6929  break;
6930  }
6931  }
6932  } // numerical zero check
6933  } // End of loop over matrices
6934  }
6935  }
6936  } // endif halo element
6937 
6938  // Time it?
6939  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
6940  {
6942  TimingHelpers::timer() - t_assemble_start;
6943  }
6944  } // End of loop over the elements
6945  } // End of vector assembly
6946 
6947 
6948  // Doc?
6949  double t_end = 0.0;
6950  double t_local = 0.0;
6951  double t_max = 0.0;
6952  double t_min = 0.0;
6953  double t_sum = 0.0;
6955  {
6956  t_end = TimingHelpers::timer();
6957  t_local = t_end - t_start;
6958  t_max = 0.0;
6959  t_min = 0.0;
6960  t_sum = 0.0;
6961  MPI_Allreduce(&t_local,
6962  &t_max,
6963  1,
6964  MPI_DOUBLE,
6965  MPI_MAX,
6966  this->communicator_pt()->mpi_comm());
6967  MPI_Allreduce(&t_local,
6968  &t_min,
6969  1,
6970  MPI_DOUBLE,
6971  MPI_MIN,
6972  this->communicator_pt()->mpi_comm());
6973  MPI_Allreduce(&t_local,
6974  &t_sum,
6975  1,
6976  MPI_DOUBLE,
6977  MPI_SUM,
6978  this->communicator_pt()->mpi_comm());
6979  double imbalance = (t_max - t_min) / (t_sum / double(nproc)) * 100.0;
6980 
6981  if (doing_residuals)
6982  {
6983  oomph_info << "\nCPU for residual computation (loc/max/min/imbal): ";
6984  }
6985  else
6986  {
6987  oomph_info << "\nCPU for Jacobian computation (loc/max/min/imbal): ";
6988  }
6989  oomph_info << t_local << " " << t_max << " " << t_min << " " << imbalance
6990  << "%\n";
6991 
6992  t_start = TimingHelpers::timer();
6993  }
6994 
6995 
6996  // Adjust number of coefficients in each row
6997  for (unsigned m = 0; m < n_matrix; m++)
6998  {
6999  unsigned max = 0;
7000  unsigned min = INT_MAX;
7001  unsigned sum = 0;
7002  unsigned sum_total = 0;
7003  for (unsigned e = 0; e < my_n_eqn; e++)
7004  {
7005  sum += ncoef[m][e];
7007  if (ncoef[m][e] > max) max = ncoef[m][e];
7008  if (ncoef[m][e] < min) min = ncoef[m][e];
7009 
7010  // Now shrink the storage to what we actually need
7011  unsigned new_allocation = ncoef[m][e];
7012  double* new_values = new double[new_allocation];
7013  unsigned* new_indices = new unsigned[new_allocation];
7014  for (unsigned c = 0; c < ncoef[m][e]; c++)
7015  {
7016  new_values[c] = matrix_values[m][e][c];
7017  new_indices[c] = matrix_col_indices[m][e][c];
7018  }
7019  delete[] matrix_values[m][e];
7020  delete[] matrix_col_indices[m][e];
7021 
7022  matrix_values[m][e] = new_values;
7023  matrix_col_indices[m][e] = new_indices;
7024  }
7025  }
7026 
7027 
7028  // Postprocess timing information and re-allocate distribution of
7029  // elements during subsequent assemblies.
7030  if ((!doing_residuals) && (!Problem_has_been_distributed) &&
7032  {
7034  }
7035 
7036  // We have determined load balancing for current setup.
7037  // This can remain the same until assign_eqn_numbers() is called
7038  // again -- the flag is re-set to true there.
7039  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
7040  {
7042  }
7043 
7044 
7045  // next we compute the number of equations and number of non-zeros to be
7046  // sent to each processor, and send/recv that information
7047  // =====================================================================
7048 
7049  // determine the number of eqns to be sent to each processor
7050  Vector<unsigned> n_eqn_for_proc(nproc, 0);
7051  Vector<unsigned> first_eqn_element_for_proc(nproc, 0);
7052  // If no equations are assembled then we don't need to do any of this
7053  if (my_n_eqn > 0)
7054  {
7055  unsigned current_p = target_dist_pt->rank_of_global_row(my_eqns[0]);
7056  first_eqn_element_for_proc[current_p] = 0;
7057  n_eqn_for_proc[current_p] = 1;
7058  for (unsigned i = 1; i < my_n_eqn; i++)
7059  {
7060  unsigned next_p = target_dist_pt->rank_of_global_row(my_eqns[i]);
7061  if (next_p != current_p)
7062  {
7063  current_p = next_p;
7064  first_eqn_element_for_proc[current_p] = i;
7065  }
7066  n_eqn_for_proc[current_p]++;
7067  }
7068  }
7069 
7070  // determine the number of non-zeros to be sent to each processor for each
7071  // matrix (if n_eqn_for_proc[p]=0, then nothing will be assembled)
7072  DenseMatrix<unsigned> nnz_for_proc(nproc, n_matrix, 0);
7073  for (unsigned p = 0; p < nproc; p++)
7074  {
7075  int first_eqn_element = first_eqn_element_for_proc[p];
7076  int last_eqn_element = (int)(first_eqn_element + n_eqn_for_proc[p]) - 1;
7077  for (unsigned m = 0; m < n_matrix; m++)
7078  {
7079  for (int i = first_eqn_element; i <= last_eqn_element; i++)
7080  {
7081  nnz_for_proc(p, m) += ncoef[m][i];
7082  }
7083  }
7084  }
7085 
7086  // next post the sends and recvs to the corresponding processors
7087  Vector<unsigned*> temp_send_storage(nproc);
7088  Vector<unsigned*> temp_recv_storage(nproc);
7089  Vector<MPI_Request> send_nnz_reqs;
7090  Vector<MPI_Request> recv_nnz_reqs;
7091  for (unsigned p = 0; p < nproc; p++)
7092  {
7093  if (p != my_rank)
7094  {
7095  temp_send_storage[p] = new unsigned[n_matrix + 1];
7096  temp_send_storage[p][0] = n_eqn_for_proc[p];
7097  for (unsigned m = 0; m < n_matrix; m++)
7098  {
7099  temp_send_storage[p][m + 1] = nnz_for_proc(p, m);
7100  }
7101  MPI_Request sreq;
7102  MPI_Isend(temp_send_storage[p],
7103  n_matrix + 1,
7104  MPI_UNSIGNED,
7105  p,
7106  0,
7107  Communicator_pt->mpi_comm(),
7108  &sreq);
7109  send_nnz_reqs.push_back(sreq);
7110  temp_recv_storage[p] = new unsigned[n_matrix + 1];
7111  MPI_Request rreq;
7112  MPI_Irecv(temp_recv_storage[p],
7113  n_matrix + 1,
7114  MPI_UNSIGNED,
7115  p,
7116  0,
7117  Communicator_pt->mpi_comm(),
7118  &rreq);
7119  recv_nnz_reqs.push_back(rreq);
7120  }
7121  }
7122 
7123  // assemble the data to be sent to each processor
7124  // ==============================================
7125 
7126  // storage
7127  Vector<unsigned*> eqns_for_proc(nproc);
7128  DenseMatrix<double*> residuals_for_proc(nproc, n_vector);
7129  DenseMatrix<unsigned*> row_start_for_proc(nproc, n_matrix);
7130  DenseMatrix<unsigned*> column_indices_for_proc(nproc, n_matrix);
7131  DenseMatrix<double*> values_for_proc(nproc, n_matrix);
7132 
7133  // equation numbers
7134  for (unsigned p = 0; p < nproc; p++)
7135  {
7136  unsigned n_eqns_p = n_eqn_for_proc[p];
7137  if (n_eqns_p > 0)
7138  {
7139  unsigned first_eqn_element = first_eqn_element_for_proc[p];
7140  unsigned first_row = target_dist_pt->first_row(p);
7141  eqns_for_proc[p] = new unsigned[n_eqns_p];
7142  for (unsigned i = 0; i < n_eqns_p; i++)
7143  {
7144  eqns_for_proc[p][i] = my_eqns[i + first_eqn_element] - first_row;
7145  }
7146  }
7147  }
7148 
7149  // residuals for p
7150  for (unsigned v = 0; v < n_vector; v++)
7151  {
7152  for (unsigned p = 0; p < nproc; p++)
7153  {
7154  unsigned n_eqns_p = n_eqn_for_proc[p];
7155  if (n_eqns_p > 0)
7156  {
7157  unsigned first_eqn_element = first_eqn_element_for_proc[p];
7158  residuals_for_proc(p, v) = new double[n_eqns_p];
7159  for (unsigned i = 0; i < n_eqns_p; i++)
7160  {
7161  residuals_for_proc(p, v)[i] =
7162  residuals_data[v][first_eqn_element + i];
7163  }
7164  }
7165  }
7166  delete[] residuals_data[v];
7167  }
7168 
7169  // matrices for p
7170  for (unsigned m = 0; m < n_matrix; m++)
7171  {
7172  for (unsigned p = 0; p < nproc; p++)
7173  {
7174  unsigned n_eqns_p = n_eqn_for_proc[p];
7175  if (n_eqns_p > 0)
7176  {
7177  unsigned first_eqn_element = first_eqn_element_for_proc[p];
7178  row_start_for_proc(p, m) = new unsigned[n_eqns_p + 1];
7179  column_indices_for_proc(p, m) = new unsigned[nnz_for_proc(p, m)];
7180  values_for_proc(p, m) = new double[nnz_for_proc(p, m)];
7181  unsigned entry = 0;
7182  for (unsigned i = 0; i < n_eqns_p; i++)
7183  {
7184  row_start_for_proc(p, m)[i] = entry;
7185  unsigned n_coef_in_row = ncoef[m][first_eqn_element + i];
7186  for (unsigned j = 0; j < n_coef_in_row; j++)
7187  {
7188  column_indices_for_proc(p, m)[entry] =
7189  matrix_col_indices[m][i + first_eqn_element][j];
7190  values_for_proc(p, m)[entry] =
7191  matrix_values[m][i + first_eqn_element][j];
7192  entry++;
7193  }
7194  }
7195  row_start_for_proc(p, m)[n_eqns_p] = entry;
7196  }
7197  }
7198  for (unsigned i = 0; i < my_n_eqn; i++)
7199  {
7200  delete[] matrix_col_indices[m][i];
7201  delete[] matrix_values[m][i];
7202  }
7203  delete[] matrix_col_indices[m];
7204  delete[] matrix_values[m];
7205  }
7206 
7207  // need to wait for the recv nnzs to complete
7208  // before we can allocate storage for the matrix recvs
7209  // ===================================================
7210 
7211  // recv and copy the datafrom the recv storage to
7212  // + nnz_from_proc
7213  // + n_eqn_from_proc
7214  Vector<MPI_Status> recv_nnz_stat(nproc - 1);
7215  MPI_Waitall(nproc - 1, &recv_nnz_reqs[0], &recv_nnz_stat[0]);
7216  Vector<unsigned> n_eqn_from_proc(nproc);
7217  DenseMatrix<unsigned> nnz_from_proc(nproc, n_matrix);
7218  for (unsigned p = 0; p < nproc; p++)
7219  {
7220  if (p != my_rank)
7221  {
7222  n_eqn_from_proc[p] = temp_recv_storage[p][0];
7223  for (unsigned m = 0; m < n_matrix; m++)
7224  {
7225  nnz_from_proc(p, m) = temp_recv_storage[p][m + 1];
7226  }
7227  delete[] temp_recv_storage[p];
7228  }
7229  else
7230  {
7231  n_eqn_from_proc[p] = n_eqn_for_proc[p];
7232  for (unsigned m = 0; m < n_matrix; m++)
7233  {
7234  nnz_from_proc(p, m) = nnz_for_proc(p, m);
7235  }
7236  }
7237  }
7238  recv_nnz_stat.clear();
7239  recv_nnz_reqs.clear();
7240 
7241  // allocate the storage for the data to be recv and post the sends recvs
7242  // =====================================================================
7243 
7244  // storage
7245  Vector<unsigned*> eqns_from_proc(nproc);
7246  DenseMatrix<double*> residuals_from_proc(nproc, n_vector);
7247  DenseMatrix<unsigned*> row_start_from_proc(nproc, n_matrix);
7248  DenseMatrix<unsigned*> column_indices_from_proc(nproc, n_matrix);
7249  DenseMatrix<double*> values_from_proc(nproc, n_matrix);
7250 
7251  // allocate and post sends and recvs
7252  double base;
7253  MPI_Aint communication_base;
7254  MPI_Get_address(&base, &communication_base);
7255  unsigned n_comm_types = 1 + 1 * n_vector + 3 * n_matrix;
7256  Vector<MPI_Request> recv_reqs;
7257  Vector<MPI_Request> send_reqs;
7258  for (unsigned p = 0; p < nproc; p++)
7259  {
7260  if (p != my_rank)
7261  {
7262  // allocate
7263  if (n_eqn_from_proc[p] > 0)
7264  {
7265  eqns_from_proc[p] = new unsigned[n_eqn_from_proc[p]];
7266  for (unsigned v = 0; v < n_vector; v++)
7267  {
7268  residuals_from_proc(p, v) = new double[n_eqn_from_proc[p]];
7269  }
7270  for (unsigned m = 0; m < n_matrix; m++)
7271  {
7272  row_start_from_proc(p, m) = new unsigned[n_eqn_from_proc[p] + 1];
7273  column_indices_from_proc(p, m) = new unsigned[nnz_from_proc(p, m)];
7274  values_from_proc(p, m) = new double[nnz_from_proc(p, m)];
7275  }
7276  }
7277 
7278  // recv
7279  if (n_eqn_from_proc[p] > 0)
7280  {
7281  MPI_Datatype types[n_comm_types];
7282  MPI_Aint offsets[n_comm_types];
7283  int count[n_comm_types];
7284  int pt = 0;
7285 
7286  // equations
7287  count[pt] = 1;
7288  MPI_Get_address(eqns_from_proc[p], &offsets[pt]);
7289  offsets[pt] -= communication_base;
7290  MPI_Type_contiguous(n_eqn_from_proc[p], MPI_UNSIGNED, &types[pt]);
7291  MPI_Type_commit(&types[pt]);
7292  pt++;
7293 
7294  // vectors
7295  for (unsigned v = 0; v < n_vector; v++)
7296  {
7297  count[pt] = 1;
7298  MPI_Get_address(residuals_from_proc(p, v), &offsets[pt]);
7299  offsets[pt] -= communication_base;
7300  MPI_Type_contiguous(n_eqn_from_proc[p], MPI_DOUBLE, &types[pt]);
7301  MPI_Type_commit(&types[pt]);
7302  pt++;
7303  }
7304 
7305  // matrices
7306  for (unsigned m = 0; m < n_matrix; m++)
7307  {
7308  // row start
7309  count[pt] = 1;
7310  MPI_Get_address(row_start_from_proc(p, m), &offsets[pt]);
7311  offsets[pt] -= communication_base;
7312  MPI_Type_contiguous(
7313  n_eqn_from_proc[p] + 1, MPI_UNSIGNED, &types[pt]);
7314  MPI_Type_commit(&types[pt]);
7315  pt++;
7316 
7317 
7318  // column indices
7319  count[pt] = 1;
7320  MPI_Get_address(column_indices_from_proc(p, m), &offsets[pt]);
7321  offsets[pt] -= communication_base;
7322  MPI_Type_contiguous(nnz_from_proc(p, m), MPI_UNSIGNED, &types[pt]);
7323  MPI_Type_commit(&types[pt]);
7324  pt++;
7325 
7326  // values
7327  count[pt] = 1;
7328  MPI_Get_address(values_from_proc(p, m), &offsets[pt]);
7329  offsets[pt] -= communication_base;
7330  MPI_Type_contiguous(nnz_from_proc(p, m), MPI_DOUBLE, &types[pt]);
7331  MPI_Type_commit(&types[pt]);
7332  pt++;
7333  }
7334 
7335  // build the combined type
7336  MPI_Datatype recv_type;
7337  MPI_Type_create_struct(
7338  n_comm_types, count, offsets, types, &recv_type);
7339  MPI_Type_commit(&recv_type);
7340  for (unsigned t = 0; t < n_comm_types; t++)
7341  {
7342  MPI_Type_free(&types[t]);
7343  }
7344  MPI_Request req;
7345  MPI_Irecv(
7346  &base, 1, recv_type, p, 1, Communicator_pt->mpi_comm(), &req);
7347  MPI_Type_free(&recv_type);
7348  recv_reqs.push_back(req);
7349  }
7350 
7351  // send
7352  if (n_eqn_for_proc[p] > 0)
7353  {
7354  MPI_Datatype types[n_comm_types];
7355  MPI_Aint offsets[n_comm_types];
7356  int count[n_comm_types];
7357  int pt = 0;
7358 
7359  // equations
7360  count[pt] = 1;
7361  MPI_Get_address(eqns_for_proc[p], &offsets[pt]);
7362  offsets[pt] -= communication_base;
7363  MPI_Type_contiguous(n_eqn_for_proc[p], MPI_UNSIGNED, &types[pt]);
7364  MPI_Type_commit(&types[pt]);
7365  pt++;
7366 
7367  // vectors
7368  for (unsigned v = 0; v < n_vector; v++)
7369  {
7370  count[pt] = 1;
7371  MPI_Get_address(residuals_for_proc(p, v), &offsets[pt]);
7372  offsets[pt] -= communication_base;
7373  MPI_Type_contiguous(n_eqn_for_proc[p], MPI_DOUBLE, &types[pt]);
7374  MPI_Type_commit(&types[pt]);
7375  pt++;
7376  }
7377 
7378  // matrices
7379  for (unsigned m = 0; m < n_matrix; m++)
7380  {
7381  // row start
7382  count[pt] = 1;
7383  MPI_Get_address(row_start_for_proc(p, m), &offsets[pt]);
7384  offsets[pt] -= communication_base;
7385  MPI_Type_contiguous(
7386  n_eqn_for_proc[p] + 1, MPI_UNSIGNED, &types[pt]);
7387  MPI_Type_commit(&types[pt]);
7388  pt++;
7389 
7390 
7391  // column indices
7392  count[pt] = 1;
7393  MPI_Get_address(column_indices_for_proc(p, m), &offsets[pt]);
7394  offsets[pt] -= communication_base;
7395  MPI_Type_contiguous(nnz_for_proc(p, m), MPI_UNSIGNED, &types[pt]);
7396  MPI_Type_commit(&types[pt]);
7397  pt++;
7398 
7399  // values
7400  count[pt] = 1;
7401  MPI_Get_address(values_for_proc(p, m), &offsets[pt]);
7402  offsets[pt] -= communication_base;
7403  MPI_Type_contiguous(nnz_for_proc(p, m), MPI_DOUBLE, &types[pt]);
7404  MPI_Type_commit(&types[pt]);
7405  pt++;
7406  }
7407 
7408  // build the combined type
7409  MPI_Datatype send_type;
7410  MPI_Type_create_struct(
7411  n_comm_types, count, offsets, types, &send_type);
7412  MPI_Type_commit(&send_type);
7413  for (unsigned t = 0; t < n_comm_types; t++)
7414  {
7415  MPI_Type_free(&types[t]);
7416  }
7417  MPI_Request req;
7418  MPI_Isend(
7419  &base, 1, send_type, p, 1, Communicator_pt->mpi_comm(), &req);
7420  MPI_Type_free(&send_type);
7421  send_reqs.push_back(req);
7422  }
7423  }
7424  // otherwise send to self
7425  else
7426  {
7427  eqns_from_proc[p] = eqns_for_proc[p];
7428  for (unsigned v = 0; v < n_vector; v++)
7429  {
7430  residuals_from_proc(p, v) = residuals_for_proc(p, v);
7431  }
7432  for (unsigned m = 0; m < n_matrix; m++)
7433  {
7434  row_start_from_proc(p, m) = row_start_for_proc(p, m);
7435  column_indices_from_proc(p, m) = column_indices_for_proc(p, m);
7436  values_from_proc(p, m) = values_for_proc(p, m);
7437  }
7438  }
7439  }
7440 
7441  // wait for the recvs to complete
7442  unsigned n_recv_req = recv_reqs.size();
7443  if (n_recv_req > 0)
7444  {
7445  Vector<MPI_Status> recv_stat(n_recv_req);
7446  MPI_Waitall(n_recv_req, &recv_reqs[0], &recv_stat[0]);
7447  }
7448 
7449  // ==============================================
7450  unsigned target_nrow_local = target_dist_pt->nrow_local();
7451 
7452  // loop over the matrices
7453  for (unsigned m = 0; m < n_matrix; m++)
7454  {
7455  // allocate row_start
7456  row_start[m] = new int[target_nrow_local + 1];
7457  row_start[m][0] = 0;
7458 
7459  // initially allocate storage based on the maximum number of non-zeros
7460  // from any one processor
7461  unsigned nnz_allocation = Parallel_sparse_assemble_previous_allocation;
7462  for (unsigned p = 0; p < nproc; p++)
7463  {
7464  nnz_allocation = std::max(nnz_allocation, nnz_from_proc(p, m));
7465  }
7466  Vector<double*> values_chunk(1);
7467  values_chunk[0] = new double[nnz_allocation];
7468  Vector<int*> column_indices_chunk(1);
7469  column_indices_chunk[0] = new int[nnz_allocation];
7470  Vector<unsigned> ncoef_in_chunk(1, 0);
7471  Vector<unsigned> size_of_chunk(1, 0);
7472  size_of_chunk[0] = nnz_allocation;
7473  unsigned current_chunk = 0;
7474 
7475  // for each row on this processor
7476  for (unsigned i = 0; i < target_nrow_local; i++)
7477  {
7478  row_start[m][i] = 0;
7479 
7480  // determine the processors that this row is on
7481  Vector<int> row_on_proc(nproc, -1);
7482  for (unsigned p = 0; p < nproc; p++)
7483  {
7484  if (n_eqn_from_proc[p] == 0)
7485  {
7486  row_on_proc[p] = -1;
7487  }
7488  else
7489  {
7490  int left = 0;
7491  int right = n_eqn_from_proc[p] - 1;
7492  int midpoint = right / 2;
7493  bool complete = false;
7494  while (!complete)
7495  {
7496  midpoint = (right + left) / 2;
7497  if (midpoint > right)
7498  {
7499  midpoint = right;
7500  }
7501  if (midpoint < left)
7502  {
7503  midpoint = left;
7504  }
7505  if (left == right)
7506  {
7507  if (eqns_from_proc[p][midpoint] == i)
7508  {
7509  midpoint = left;
7510  }
7511  else
7512  {
7513  midpoint = -1;
7514  }
7515  complete = true;
7516  }
7517  else if (eqns_from_proc[p][midpoint] == i)
7518  {
7519  complete = true;
7520  }
7521  else if (eqns_from_proc[p][midpoint] > i)
7522  {
7523  right = std::max(midpoint - 1, left);
7524  }
7525  else
7526  {
7527  left = std::min(midpoint + 1, right);
7528  }
7529  }
7530  row_on_proc[p] = midpoint;
7531  }
7532  }
7533 
7534  // for each processor build this row of the matrix
7535  unsigned check_first = ncoef_in_chunk[current_chunk];
7536  unsigned check_last = check_first;
7537  for (unsigned p = 0; p < nproc; p++)
7538  {
7539  if (row_on_proc[p] != -1)
7540  {
7541  int row = row_on_proc[p];
7542  unsigned first = row_start_from_proc(p, m)[row];
7543  unsigned last = row_start_from_proc(p, m)[row + 1];
7544  for (unsigned l = first; l < last; l++)
7545  {
7546  bool done = false;
7547  for (unsigned j = check_first; j <= check_last && !done; j++)
7548  {
7549  if (j == check_last)
7550  {
7551  // is this temp array full, do we need to allocate
7552  // a new temp array
7553  if (ncoef_in_chunk[current_chunk] ==
7554  size_of_chunk[current_chunk])
7555  {
7556  // number of chunks allocated
7557  unsigned n_chunk = values_chunk.size();
7558 
7559  // determine the number of non-zeros added so far
7560  // (excluding the current row)
7561  unsigned nnz_so_far = 0;
7562  for (unsigned c = 0; c < n_chunk; c++)
7563  {
7564  nnz_so_far += ncoef_in_chunk[c];
7565  }
7566  nnz_so_far -= row_start[m][i];
7567 
7568  // average number of non-zeros per row
7569  unsigned avg_nnz = nnz_so_far / (i + 1);
7570 
7571  // number of rows left +1
7572  unsigned nrows_left = target_nrow_local - i;
7573 
7574  // allocation for next chunk
7575  unsigned next_chunk_size =
7576  avg_nnz * nrows_left + row_start[m][i];
7577 
7578  // allocate storage in next chunk
7579  current_chunk++;
7580  n_chunk++;
7581  values_chunk.resize(n_chunk);
7582  values_chunk[current_chunk] = new double[next_chunk_size];
7583  column_indices_chunk.resize(n_chunk);
7584  column_indices_chunk[current_chunk] =
7585  new int[next_chunk_size];
7586  size_of_chunk.resize(n_chunk);
7587  size_of_chunk[current_chunk] = next_chunk_size;
7588  ncoef_in_chunk.resize(n_chunk);
7589 
7590  // copy current row from previous chunk to new chunk
7591  for (unsigned k = check_first; k < check_last; k++)
7592  {
7593  values_chunk[current_chunk][k - check_first] =
7594  values_chunk[current_chunk - 1][k];
7595  column_indices_chunk[current_chunk][k - check_first] =
7596  column_indices_chunk[current_chunk - 1][k];
7597  }
7598  ncoef_in_chunk[current_chunk - 1] -= row_start[m][i];
7599  ncoef_in_chunk[current_chunk] = row_start[m][i];
7600 
7601  // update first_check and last_check
7602  check_first = 0;
7603  check_last = row_start[m][i];
7604  j = check_last;
7605  }
7606 
7607  // add the coefficient
7608  values_chunk[current_chunk][j] = values_from_proc(p, m)[l];
7609  column_indices_chunk[current_chunk][j] =
7610  column_indices_from_proc(p, m)[l];
7611  ncoef_in_chunk[current_chunk]++;
7612  row_start[m][i]++;
7613  check_last++;
7614  done = true;
7615  }
7616  else if (column_indices_chunk[current_chunk][j] ==
7617  (int)column_indices_from_proc(p, m)[l])
7618  {
7619  values_chunk[current_chunk][j] += values_from_proc(p, m)[l];
7620  done = true;
7621  }
7622  }
7623  }
7624  }
7625  }
7626  }
7627 
7628  // delete recv data for this matrix
7629  for (unsigned p = 0; p < nproc; p++)
7630  {
7631  if (n_eqn_from_proc[p] > 0)
7632  {
7633  delete[] row_start_from_proc(p, m);
7634  delete[] column_indices_from_proc(p, m);
7635  delete[] values_from_proc(p, m);
7636  }
7637  }
7638 
7639  // next we take the chunk base storage of the column indices and values
7640  // and copy into a single contiguous block of memory
7641  // ====================================================================
7642  unsigned n_chunk = values_chunk.size();
7643  nnz[m] = 0;
7644  for (unsigned c = 0; c < n_chunk; c++)
7645  {
7646  nnz[m] += ncoef_in_chunk[c];
7647  }
7649 
7650  // allocate
7651  values[m] = new double[nnz[m]];
7652  column_indices[m] = new int[nnz[m]];
7653 
7654  // copy
7655  unsigned pt = 0;
7656  for (unsigned c = 0; c < n_chunk; c++)
7657  {
7658  unsigned nc = ncoef_in_chunk[c];
7659  for (unsigned i = 0; i < nc; i++)
7660  {
7661  values[m][pt + i] = values_chunk[c][i];
7662  column_indices[m][pt + i] = column_indices_chunk[c][i];
7663  }
7664  pt += nc;
7665  delete[] values_chunk[c];
7666  delete[] column_indices_chunk[c];
7667  }
7668 
7669  // the row_start vector currently contains the number of coefs in each
7670  // row. Update
7671  // ===================================================================
7672  unsigned g = row_start[m][0];
7673  row_start[m][0] = 0;
7674  for (unsigned i = 1; i < target_nrow_local; i++)
7675  {
7676  unsigned h = g + row_start[m][i];
7677  row_start[m][i] = g;
7678  g = h;
7679  }
7680  row_start[m][target_nrow_local] = g;
7681  }
7682 
7683  // next accumulate the residuals
7684  for (unsigned v = 0; v < n_vector; v++)
7685  {
7686  residuals[v] = new double[target_nrow_local];
7687  for (unsigned i = 0; i < target_nrow_local; i++)
7688  {
7689  residuals[v][i] = 0;
7690  }
7691  for (unsigned p = 0; p < nproc; p++)
7692  {
7693  if (n_eqn_from_proc[p] > 0)
7694  {
7695  unsigned n_eqn_p = n_eqn_from_proc[p];
7696  for (unsigned i = 0; i < n_eqn_p; i++)
7697  {
7698  residuals[v][eqns_from_proc[p][i]] += residuals_from_proc(p, v)[i];
7699  }
7700  delete[] residuals_from_proc(p, v);
7701  }
7702  }
7703  }
7704 
7705  // delete list of eqns from proc
7706  for (unsigned p = 0; p < nproc; p++)
7707  {
7708  if (n_eqn_from_proc[p] > 0)
7709  {
7710  delete[] eqns_from_proc[p];
7711  }
7712  }
7713 
7714  // and wait for sends to complete
7715  Vector<MPI_Status> send_nnz_stat(nproc - 1);
7716  MPI_Waitall(nproc - 1, &send_nnz_reqs[0], &send_nnz_stat[0]);
7717  for (unsigned p = 0; p < nproc; p++)
7718  {
7719  if (p != my_rank)
7720  {
7721  delete[] temp_send_storage[p];
7722  }
7723  }
7724  send_nnz_stat.clear();
7725  send_nnz_reqs.clear();
7726 
7727  // wait for the matrix data sends to complete and delete the data
7728  unsigned n_send_reqs = send_reqs.size();
7729  if (n_send_reqs > 0)
7730  {
7731  Vector<MPI_Status> send_stat(n_send_reqs);
7732  MPI_Waitall(n_send_reqs, &send_reqs[0], &send_stat[0]);
7733  for (unsigned p = 0; p < nproc; p++)
7734  {
7735  if (p != my_rank)
7736  {
7737  if (n_eqn_for_proc[p])
7738  {
7739  delete[] eqns_for_proc[p];
7740  for (unsigned m = 0; m < n_matrix; m++)
7741  {
7742  delete[] row_start_for_proc(p, m);
7743  delete[] column_indices_for_proc(p, m);
7744  delete[] values_for_proc(p, m);
7745  }
7746  for (unsigned v = 0; v < n_vector; v++)
7747  {
7748  delete[] residuals_for_proc(p, v);
7749  }
7750  }
7751  }
7752  }
7753  }
7754 
7755  // Doc?
7757  {
7758  t_end = TimingHelpers::timer();
7759  t_local = t_end - t_start;
7760  t_max = 0.0;
7761  t_min = 0.0;
7762  t_sum = 0.0;
7763  MPI_Allreduce(&t_local,
7764  &t_max,
7765  1,
7766  MPI_DOUBLE,
7767  MPI_MAX,
7768  this->communicator_pt()->mpi_comm());
7769  MPI_Allreduce(&t_local,
7770  &t_min,
7771  1,
7772  MPI_DOUBLE,
7773  MPI_MIN,
7774  this->communicator_pt()->mpi_comm());
7775  MPI_Allreduce(&t_local,
7776  &t_sum,
7777  1,
7778  MPI_DOUBLE,
7779  MPI_SUM,
7780  this->communicator_pt()->mpi_comm());
7781  double imbalance = (t_max - t_min) / (t_sum / double(nproc)) * 100.0;
7782  if (doing_residuals)
7783  {
7784  oomph_info << "CPU for residual distribut. (loc/max/min/imbal): ";
7785  }
7786  else
7787  {
7788  oomph_info << "CPU for Jacobian distribut. (loc/max/min/imbal): ";
7789  }
7790  oomph_info << t_local << " " << t_max << " " << t_min << " " << imbalance
7791  << "%\n\n";
7792  }
7793  }
7794 
7795 #endif
7796 
7797 
7798  //================================================================
7799  /// Get the full Jacobian by finite differencing
7800  //================================================================
7802  DenseMatrix<double>& jacobian)
7803  {
7804 #ifdef OOMPH_HAS_MPI
7805 
7807  {
7808  OomphLibWarning("This is unlikely to work with a distributed problem",
7809  " Problem::get_fd_jacobian()",
7810  OOMPH_EXCEPTION_LOCATION);
7811  }
7812 #endif
7813 
7814 
7815  // Find number of dofs
7816  const unsigned long n_dof = ndof();
7817 
7818  // Advanced residuals
7819  DoubleVector residuals_pls;
7820 
7821  // Get reference residuals
7822  get_residuals(residuals);
7823 
7824  const double FD_step = 1.0e-8;
7825 
7826  // Make sure the Jacobian is the right size (since we don't care about
7827  // speed).
7828  jacobian.resize(n_dof, n_dof);
7829 
7830  // Loop over all dofs
7831  for (unsigned long jdof = 0; jdof < n_dof; jdof++)
7832  {
7833  double backup = *Dof_pt[jdof];
7834  *Dof_pt[jdof] += FD_step;
7835 
7836  // We're checking if the new values for Dof_pt[] actually
7837  // solve the entire problem --> update as if problem had
7838  // been solved
7842 
7843  // Get advanced residuals
7844  get_residuals(residuals_pls);
7845 
7846  for (unsigned long ieqn = 0; ieqn < n_dof; ieqn++)
7847  {
7848  jacobian(ieqn, jdof) =
7849  (residuals_pls[ieqn] - residuals[ieqn]) / FD_step;
7850  }
7851 
7852  *Dof_pt[jdof] = backup;
7853  }
7854 
7855  // Reset problem to state it was in
7859  }
7860 
7861  //======================================================================
7862  /// Get derivative of the residuals vector wrt a global parameter
7863  /// This is required in continuation problems
7864  //=======================================================================
7865  void Problem::get_derivative_wrt_global_parameter(double* const& parameter_pt,
7866  DoubleVector& result)
7867  {
7868  // If we are doing the calculation analytically then call the appropriate
7869  // handler and then calling get_residuals
7870  if (is_dparameter_calculated_analytically(parameter_pt))
7871  {
7872  // Locally cache pointer to assembly handler
7873  AssemblyHandler* const old_assembly_handler_pt = Assembly_handler_pt;
7874  // Create a new assembly handler that replaces get_residuals by
7875  // get_dresiduals_dparameter for each element
7877  new ParameterDerivativeHandler(old_assembly_handler_pt, parameter_pt);
7878  // Get the residuals, which will be dresiduals by dparameter
7879  this->get_residuals(result);
7880  // Delete the parameter derivative handler
7881  delete Assembly_handler_pt;
7882  // Reset the assembly handler to the original handler
7883  Assembly_handler_pt = old_assembly_handler_pt;
7884 
7885  /*AssemblyHandler* const assembly_handler_pt = Assembly_handler_pt;
7886  //Loop over all the elements
7887  unsigned long Element_pt_range = Mesh_pt->nelement();
7888  for(unsigned long e=0;e<Element_pt_range;e++)
7889  {
7890  //Get the pointer to the element
7891  GeneralisedElement* elem_pt = Mesh_pt->element_pt(e);
7892  //Find number of dofs in the element
7893  unsigned n_element_dofs = assembly_handler_pt->ndof(elem_pt);
7894  //Set up an array
7895  Vector<double> element_residuals(n_element_dofs);
7896  //Fill the array
7897  assembly_handler_pt->get_dresiduals_dparameter(elem_pt,parameter_pt,
7898  element_residuals);
7899  //Now loop over the dofs and assign values to global Vector
7900  for(unsigned l=0;l<n_element_dofs;l++)
7901  {
7902  result[assembly_handler_pt->eqn_number(elem_pt,l)]
7903  += element_residuals[l];
7904  }
7905  }*/
7906 
7907  // for(unsigned n=0;n<n_dof;n++)
7908  // {std::cout << "BLA " << n << " " << result[n] << "\n";}
7909  }
7910  // Otherwise use the finite difference default
7911  else
7912  {
7913  // Get the (global) residuals and store in the result vector
7914  get_residuals(result);
7915 
7916  // Storage for the new residuals
7917  DoubleVector newres;
7918 
7919  // Increase the global parameter
7920  const double FD_step = 1.0e-8;
7921 
7922  // Store the current value of the parameter
7923  double param_value = *parameter_pt;
7924 
7925  // Increase the parameter
7926  *parameter_pt += FD_step;
7927 
7928  // Do any possible updates
7930 
7931  // Get the new residuals
7932  get_residuals(newres);
7933 
7934  // Find the number of local rows
7935  //(I think it's a global vector, so that should be fine)
7936  const unsigned ndof_local = result.nrow_local();
7937 
7938  // Do the finite differencing in the local variables
7939  for (unsigned n = 0; n < ndof_local; ++n)
7940  {
7941  result[n] = (newres[n] - result[n]) / FD_step;
7942  }
7943 
7944  // Reset the value of the parameter
7945  *parameter_pt = param_value;
7946 
7947  // Do any possible updates
7949  }
7950  }
7951 
7952 
7953  //======================================================================
7954  /// Return the product of the global hessian (derivative of Jacobian
7955  /// matrix with respect to all variables) with
7956  /// an eigenvector, Y, and any number of other specified vectors C
7957  /// (d(J_{ij})/d u_{k}) Y_{j} C_{k}.
7958  /// This function is used in assembling and solving the augmented systems
7959  /// associated with bifurcation tracking.
7960  /// The default implementation is to use finite differences at the global
7961  /// level.
7962  //========================================================================
7964  DoubleVectorWithHaloEntries const& Y,
7967  {
7968  // How many vector products must we construct
7969  const unsigned n_vec = C.size();
7970 
7971  // currently only global (non-distributed) distributions are allowed
7972  // LinearAlgebraDistribution* dist_pt = new
7973  // LinearAlgebraDistribution(Communicator_pt,n_dof,false);
7974 
7975  // Cache the assembly hander
7977 
7978  // Rebuild the results vectors and initialise to zero
7979  // use the same distribution of the vector Y
7980  for (unsigned i = 0; i < n_vec; i++)
7981  {
7982  product[i].build(Y.distribution_pt(), 0.0);
7983  product[i].initialise(0.0);
7984  }
7985 
7986 // Setup the halo schemes for the result
7987 #ifdef OOMPH_HAS_MPI
7989  {
7990  for (unsigned i = 0; i < n_vec; i++)
7991  {
7992  product[i].build_halo_scheme(this->Halo_scheme_pt);
7993  }
7994  }
7995 #endif
7996 
7997  // If we are doing the calculation analytically then call the appropriate
7998  // handler
7999  // A better way to do this is probably to hook into the get_residuals
8000  // framework but with a different member function of the assembly
8001  // handler
8003  {
8004  // Loop over all the elements
8005  unsigned long Element_pt_range = Mesh_pt->nelement();
8006  for (unsigned long e = 0; e < Element_pt_range; e++)
8007  {
8008  // Get the pointer to the element
8009  GeneralisedElement* elem_pt = Mesh_pt->element_pt(e);
8010 // Do not loop over halo elements
8011 #ifdef OOMPH_HAS_MPI
8012  if (!elem_pt->is_halo())
8013  {
8014 #endif
8015  // Find number of dofs in the element
8016  unsigned n_var = assembly_handler_pt->ndof(elem_pt);
8017  // Set up a matrix for the input and output
8018  Vector<double> Y_local(n_var);
8019  DenseMatrix<double> C_local(n_vec, n_var);
8020  DenseMatrix<double> product_local(n_vec, n_var);
8021 
8022  // Translate the global input vectors into the local storage
8023  // Probably horribly inefficient, but otherwise things get really
8024  // messy at the elemental level
8025  for (unsigned l = 0; l < n_var; l++)
8026  {
8027  // Cache the global equation number
8028  const unsigned long eqn_number =
8029  assembly_handler_pt->eqn_number(elem_pt, l);
8030 
8031  Y_local[l] = Y.global_value(eqn_number);
8032  for (unsigned i = 0; i < n_vec; i++)
8033  {
8034  C_local(i, l) = C[i].global_value(eqn_number);
8035  }
8036  }
8037 
8038  // Fill the array
8040  elem_pt, Y_local, C_local, product_local);
8041 
8042  // Assign the local results to the global vector
8043  for (unsigned l = 0; l < n_var; l++)
8044  {
8045  const unsigned long eqn_number =
8046  assembly_handler_pt->eqn_number(elem_pt, l);
8047 
8048  for (unsigned i = 0; i < n_vec; i++)
8049  {
8050  product[i].global_value(eqn_number) += product_local(i, l);
8051  // std::cout << "BLA " << e << " " << i << " "
8052  // << l << " " << product_local(i,l) << "\n";
8053  }
8054  }
8055 #ifdef OOMPH_HAS_MPI
8056  }
8057 #endif
8058  }
8059  }
8060  // Otherwise calculate using finite differences by
8061  // perturbing the jacobian along a particular direction
8062  else
8063  {
8064  // Cache the finite difference step
8065  /// Alice: My bifurcation tracking converges better with this FD_step
8066  /// as 1.0e-5. The default value remains at 1.0e-8.
8068 
8069  // We can now construct our multipliers
8070  const unsigned n_dof_local = this->Dof_distribution_pt->nrow_local();
8071  // Prepare to scale
8072  double dof_length = 0.0;
8073  Vector<double> C_length(n_vec, 0.0);
8074 
8075  for (unsigned n = 0; n < n_dof_local; n++)
8076  {
8077  if (std::fabs(this->dof(n)) > dof_length)
8078  {
8079  dof_length = std::fabs(this->dof(n));
8080  }
8081  }
8082 
8083  // C is assumed to have the same distribution as the dofs
8084  for (unsigned i = 0; i < n_vec; i++)
8085  {
8086  for (unsigned n = 0; n < n_dof_local; n++)
8087  {
8088  if (std::fabs(C[i][n]) > C_length[i])
8089  {
8090  C_length[i] = std::fabs(C[i][n]);
8091  }
8092  }
8093  }
8094 
8095  // Now broadcast the information, if distributed
8096 #ifdef OOMPH_HAS_MPI
8098  {
8099  const unsigned n_length = n_vec + 1;
8100  double all_length[n_length];
8101  all_length[0] = dof_length;
8102  for (unsigned i = 0; i < n_vec; i++)
8103  {
8104  all_length[i + 1] = C_length[i];
8105  }
8106 
8107  // Do the MPI call
8108  double all_length_reduce[n_length];
8109  MPI_Allreduce(all_length,
8110  all_length_reduce,
8111  n_length,
8112  MPI_DOUBLE,
8113  MPI_MAX,
8114  this->communicator_pt()->mpi_comm());
8115 
8116  // Read out the information
8117  dof_length = all_length_reduce[0];
8118  for (unsigned i = 0; i < n_vec; i++)
8119  {
8120  C_length[i] = all_length_reduce[i + 1];
8121  }
8122  }
8123 #endif
8124 
8125  // Form the multipliers
8126  Vector<double> C_mult(n_vec, 0.0);
8127  for (unsigned i = 0; i < n_vec; i++)
8128  {
8129  C_mult[i] = dof_length / C_length[i];
8130  C_mult[i] += FD_step;
8131  C_mult[i] *= FD_step;
8132  }
8133 
8134 
8135  // Dummy vector to stand in the place of the residuals
8136  Vector<double> dummy_res;
8137 
8138  // Calculate the product of the jacobian matrices, etc by looping over the
8139  // elements
8140  const unsigned long n_element = this->mesh_pt()->nelement();
8141  for (unsigned long e = 0; e < n_element; e++)
8142  {
8143  GeneralisedElement* elem_pt = this->mesh_pt()->element_pt(e);
8144  // Ignore halo's of course
8145 #ifdef OOMPH_HAS_MPI
8146  if (!elem_pt->is_halo())
8147  {
8148 #endif
8149  // Loop over the ndofs in each element
8150  unsigned n_var = assembly_handler_pt->ndof(elem_pt);
8151  // Resize the dummy residuals vector
8152  dummy_res.resize(n_var);
8153  // Allocate storage for the unperturbed jacobian matrix
8154  DenseMatrix<double> jac(n_var);
8155  // Get unperturbed jacobian
8156  assembly_handler_pt->get_jacobian(elem_pt, dummy_res, jac);
8157 
8158  // Backup the dofs
8159  Vector<double> dof_bac(n_var);
8160  for (unsigned n = 0; n < n_var; n++)
8161  {
8162  unsigned eqn_number = assembly_handler_pt->eqn_number(elem_pt, n);
8163  dof_bac[n] = *this->global_dof_pt(eqn_number);
8164  }
8165 
8166  // Now loop over all vectors C
8167  for (unsigned i = 0; i < n_vec; i++)
8168  {
8169  // Perturb the dofs by the appropriate vector
8170  for (unsigned n = 0; n < n_var; n++)
8171  {
8172  unsigned eqn_number = assembly_handler_pt->eqn_number(elem_pt, n);
8173  // Perturb by vector C[i]
8174  *this->global_dof_pt(eqn_number) +=
8175  C_mult[i] * C[i].global_value(eqn_number);
8176  }
8178 
8179  // Allocate storage for the perturbed jacobian
8180  DenseMatrix<double> jac_C(n_var);
8181 
8182  // Now get the new jacobian
8183  assembly_handler_pt->get_jacobian(elem_pt, dummy_res, jac_C);
8184 
8185  // Reset the dofs
8186  for (unsigned n = 0; n < n_var; n++)
8187  {
8188  unsigned eqn_number = assembly_handler_pt->eqn_number(elem_pt, n);
8189  *this->global_dof_pt(eqn_number) = dof_bac[n];
8190  }
8192 
8193  // Now work out the products
8194  for (unsigned n = 0; n < n_var; n++)
8195  {
8196  unsigned eqn_number = assembly_handler_pt->eqn_number(elem_pt, n);
8197  double prod_c = 0.0;
8198  for (unsigned m = 0; m < n_var; m++)
8199  {
8200  unsigned unknown = assembly_handler_pt->eqn_number(elem_pt, m);
8201  prod_c += (jac_C(n, m) - jac(n, m)) * Y.global_value(unknown);
8202  }
8203  // std::cout << "FD " << e << " " << i << " "
8204  // << n << " " << prod_c/C_mult[i] << "\n";
8205  product[i].global_value(eqn_number) += prod_c / C_mult[i];
8206  }
8207  }
8208 #ifdef OOMPH_HAS_MPI
8209  }
8210 #endif
8211  } // End of loop over elements
8212  }
8213 
8214  // If we have a distributed problem then gather all
8215  // values
8216 #ifdef OOMPH_HAS_MPI
8218  {
8219  // Sum all values if distributed
8220  for (unsigned i = 0; i < n_vec; i++)
8221  {
8222  product[i].sum_all_halo_and_haloed_values();
8223  }
8224  }
8225 #endif
8226  }
8227 
8228 
8229  //==================================================================
8230  /// Solve the eigenproblem
8231  //==================================================================
8232  void Problem::solve_eigenproblem(const unsigned& n_eval,
8233  Vector<std::complex<double>>& alpha,
8234  Vector<double>& beta,
8235  Vector<DoubleVector>& eigenvector_real,
8236  Vector<DoubleVector>& eigenvector_imag,
8237  const bool& make_timesteppers_steady)
8238  {
8239  // If the boolean flag is steady, then make all the timesteppers steady
8240  // before solving the eigenproblem. This will "switch off" the
8241  // time-derivative terms in the jacobian matrix
8242  if (make_timesteppers_steady)
8243  {
8244  // Find out how many timesteppers there are
8245  const unsigned n_time_steppers = ntime_stepper();
8246 
8247  // Vector of bools to store the is_steady status of the various
8248  // timesteppers when we came in here
8249  std::vector<bool> was_steady(n_time_steppers);
8250 
8251  // Loop over them all and make them (temporarily) static
8252  for (unsigned i = 0; i < n_time_steppers; i++)
8253  {
8254  was_steady[i] = time_stepper_pt(i)->is_steady();
8256  }
8257 
8258  const bool do_adjoint_problem = false;
8259  // Call the Eigenproblem for the eigensolver
8261  n_eval,
8262  alpha,
8263  beta,
8264  eigenvector_real,
8265  eigenvector_imag,
8266  do_adjoint_problem);
8267 
8268  // Reset the is_steady status of all timesteppers that
8269  // weren't already steady when we came in here and reset their
8270  // weights
8271  for (unsigned i = 0; i < n_time_steppers; i++)
8272  {
8273  if (!was_steady[i])
8274  {
8276  }
8277  }
8278  }
8279  // Otherwise if we don't want to make the problem steady, just
8280  // assemble and solve the eigensystem
8281  else
8282  {
8283  const bool do_adjoint_problem = false;
8284  // Call the Eigenproblem for the eigensolver
8286  n_eval,
8287  alpha,
8288  beta,
8289  eigenvector_real,
8290  eigenvector_imag,
8291  do_adjoint_problem);
8292  }
8293  }
8294 
8295 
8296  //==================================================================
8297  /// Solve the eigenproblem
8298  //==================================================================
8299  void Problem::solve_eigenproblem(const unsigned& n_eval,
8300  Vector<std::complex<double>>& eigenvalue,
8301  Vector<DoubleVector>& eigenvector_real,
8302  Vector<DoubleVector>& eigenvector_imag,
8303  const bool& make_timesteppers_steady)
8304  {
8305  // If the boolean flag is steady, then make all the timesteppers steady
8306  // before solving the eigenproblem. This will "switch off" the
8307  // time-derivative terms in the jacobian matrix
8308  if (make_timesteppers_steady)
8309  {
8310  // Find out how many timesteppers there are
8311  const unsigned n_time_steppers = ntime_stepper();
8312 
8313  // Vector of bools to store the is_steady status of the various
8314  // timesteppers when we came in here
8315  std::vector<bool> was_steady(n_time_steppers);
8316 
8317  // Loop over them all and make them (temporarily) static
8318  for (unsigned i = 0; i < n_time_steppers; i++)
8319  {
8320  was_steady[i] = time_stepper_pt(i)->is_steady();
8322  }
8323 
8324  const bool do_adjoint_problem = false;
8325  // Call the Eigenproblem for the eigensolver
8327  n_eval,
8328  eigenvalue,
8329  eigenvector_real,
8330  eigenvector_imag,
8331  do_adjoint_problem);
8332 
8333  // Reset the is_steady status of all timesteppers that
8334  // weren't already steady when we came in here and reset their
8335  // weights
8336  for (unsigned i = 0; i < n_time_steppers; i++)
8337  {
8338  if (!was_steady[i])
8339  {
8341  }
8342  }
8343  }
8344  // Otherwise if we don't want to make the problem steady, just
8345  // assemble and solve the eigensystem
8346  else
8347  {
8348  const bool do_adjoint_problem = false;
8349  // Call the Eigenproblem for the eigensolver
8351  n_eval,
8352  eigenvalue,
8353  eigenvector_real,
8354  eigenvector_imag,
8355  do_adjoint_problem);
8356  }
8357  }
8358 
8359 
8360  //==================================================================
8361  /// Solve the adjoint eigenproblem
8362  //==================================================================
8364  const unsigned& n_eval,
8365  Vector<std::complex<double>>& eigenvalue,
8366  Vector<DoubleVector>& eigenvector_real,
8367  Vector<DoubleVector>& eigenvector_imag,
8368  const bool& make_timesteppers_steady)
8369  {
8370  // If the boolean flag is steady, then make all the timesteppers steady
8371  // before solving the eigenproblem. This will "switch off" the
8372  // time-derivative terms in the jacobian matrix
8373  if (make_timesteppers_steady)
8374  {
8375  // Find out how many timesteppers there are
8376  const unsigned n_time_steppers = ntime_stepper();
8377 
8378  // Vector of bools to store the is_steady status of the various
8379  // timesteppers when we came in here
8380  std::vector<bool> was_steady(n_time_steppers);
8381 
8382  // Loop over them all and make them (temporarily) static
8383  for (unsigned i = 0; i < n_time_steppers; i++)
8384  {
8385  was_steady[i] = time_stepper_pt(i)->is_steady();
8387  }
8388 
8389  const bool do_adjoint_problem = true;
8390  // Call the Eigenproblem for the eigensolver
8392  n_eval,
8393  eigenvalue,
8394  eigenvector_real,
8395  eigenvector_imag,
8396  do_adjoint_problem);
8397 
8398  // Reset the is_steady status of all timesteppers that
8399  // weren't already steady when we came in here and reset their
8400  // weights
8401  for (unsigned i = 0; i < n_time_steppers; i++)
8402  {
8403  if (!was_steady[i])
8404  {
8406  }
8407  }
8408  }
8409  // Otherwise if we don't want to make the problem steady, just
8410  // assemble and solve the eigensystem
8411  else
8412  {
8413  const bool do_adjoint_problem = true;
8414  // Call the Eigenproblem for the eigensolver
8416  n_eval,
8417  eigenvalue,
8418  eigenvector_real,
8419  eigenvector_imag,
8420  do_adjoint_problem);
8421  }
8422  }
8423 
8424  //===================================================================
8425  /// Get the matrices required to solve an eigenproblem
8426  /// WARNING: temporarily this method only works with non-distributed
8427  /// matrices
8428  //===================================================================
8430  CRDoubleMatrix& main_matrix,
8431  const double& shift)
8432  {
8433  // Three different cases again here:
8434  // 1) Compiled with MPI, but run in serial
8435  // 2) Compiled with MPI, but MPI not initialised in driver
8436  // 3) Serial version
8437 
8438 
8439 #ifdef PARANOID
8440  if (mass_matrix.distribution_built() && main_matrix.distribution_built())
8441  {
8442  // Check that the distribution of the mass matrix and jacobian match
8443  if (!(*mass_matrix.distribution_pt() == *main_matrix.distribution_pt()))
8444  {
8445  std::ostringstream error_stream;
8446  error_stream
8447  << "The distributions of the jacobian and mass matrix are\n"
8448  << "not the same and they must be.\n";
8449  throw OomphLibError(
8450  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
8451  }
8452 
8453  if (mass_matrix.nrow() != this->ndof())
8454  {
8455  std::ostringstream error_stream;
8456  error_stream
8457  << "mass_matrix has a distribution, but the number of rows is not "
8458  << "equal to the number of degrees of freedom in the problem.";
8459  throw OomphLibError(
8460  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
8461  }
8462 
8463  if (main_matrix.nrow() != this->ndof())
8464  {
8465  std::ostringstream error_stream;
8466  error_stream
8467  << "main_matrix has a distribution, but the number of rows is not "
8468  << "equal to the number of degrees of freedom in the problem.";
8469  throw OomphLibError(
8470  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
8471  }
8472  }
8473  // If the distributions are not the same, then complain
8474  else if (main_matrix.distribution_built() !=
8475  mass_matrix.distribution_built())
8476  {
8477  std::ostringstream error_stream;
8478  error_stream << "The distribution of the jacobian and mass matrix must "
8479  << "both be setup or both not setup";
8480  throw OomphLibError(
8481  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
8482  }
8483 #endif
8484 
8485  // Store the old assembly handler
8486  AssemblyHandler* old_assembly_handler_pt = Assembly_handler_pt;
8487  // Now setup the eigenproblem handler, pass in the value of the shift
8489 
8490  // Prepare the storage formats.
8491  Vector<int*> column_or_row_index(2);
8492  Vector<int*> row_or_column_start(2);
8493  Vector<double*> value(2);
8494  Vector<unsigned> nnz(2);
8495  // Allocate pointer to residuals, although not used in these problems
8496  Vector<double*> residuals_vectors(0);
8497 
8498  // determine the distribution for the jacobian (main matrix)
8499  // IF the jacobian has distribution setup then use that
8500  // ELSE determine the distribution based on the
8501  // distributed_matrix_distribution enum
8502  LinearAlgebraDistribution* dist_pt = 0;
8503  if (main_matrix.distribution_built())
8504  {
8505  dist_pt = new LinearAlgebraDistribution(main_matrix.distribution_pt());
8506  }
8507  else
8508  {
8510  }
8511 
8512 
8513  // The matrix is in compressed row format
8514  bool compressed_row_flag = true;
8515 
8516 #ifdef OOMPH_HAS_MPI
8517  //
8518  if (Communicator_pt->nproc() == 1)
8519  {
8520 #endif
8521 
8522  sparse_assemble_row_or_column_compressed(column_or_row_index,
8523  row_or_column_start,
8524  value,
8525  nnz,
8526  residuals_vectors,
8527  compressed_row_flag);
8528 
8529  // The main matrix is the first entry
8530  main_matrix.build(dist_pt);
8531  main_matrix.build_without_copy(dist_pt->nrow(),
8532  nnz[0],
8533  value[0],
8534  column_or_row_index[0],
8535  row_or_column_start[0]);
8536  // The mass matrix is the second entry
8537  mass_matrix.build(dist_pt);
8538  mass_matrix.build_without_copy(dist_pt->nrow(),
8539  nnz[1],
8540  value[1],
8541  column_or_row_index[1],
8542  row_or_column_start[1]);
8543 #ifdef OOMPH_HAS_MPI
8544  }
8545  else
8546  {
8547  if (dist_pt->distributed())
8548  {
8549  parallel_sparse_assemble(dist_pt,
8550  column_or_row_index,
8551  row_or_column_start,
8552  value,
8553  nnz,
8554  residuals_vectors);
8555  // The main matrix is the first entry
8556  main_matrix.build(dist_pt);
8557  main_matrix.build_without_copy(dist_pt->nrow(),
8558  nnz[0],
8559  value[0],
8560  column_or_row_index[0],
8561  row_or_column_start[0]);
8562  // The mass matrix is the second entry
8563  mass_matrix.build(dist_pt);
8564  mass_matrix.build_without_copy(dist_pt->nrow(),
8565  nnz[1],
8566  value[1],
8567  column_or_row_index[1],
8568  row_or_column_start[1]);
8569  }
8570  else
8571  {
8572  LinearAlgebraDistribution* temp_dist_pt =
8573  new LinearAlgebraDistribution(Communicator_pt, dist_pt->nrow(), true);
8574  parallel_sparse_assemble(temp_dist_pt,
8575  column_or_row_index,
8576  row_or_column_start,
8577  value,
8578  nnz,
8579  residuals_vectors);
8580  // The main matrix is the first entry
8581  main_matrix.build(temp_dist_pt);
8582  main_matrix.build_without_copy(dist_pt->nrow(),
8583  nnz[0],
8584  value[0],
8585  column_or_row_index[0],
8586  row_or_column_start[0]);
8587  main_matrix.redistribute(dist_pt);
8588  // The mass matrix is the second entry
8589  mass_matrix.build(temp_dist_pt);
8590  mass_matrix.build_without_copy(dist_pt->nrow(),
8591  nnz[1],
8592  value[1],
8593  column_or_row_index[1],
8594  row_or_column_start[1]);
8595  mass_matrix.redistribute(dist_pt);
8596  delete temp_dist_pt;
8597  }
8598  }
8599 #endif
8600 
8601  // clean up dist_pt and residuals_vector pt
8602  delete dist_pt;
8603 
8604  // Delete the eigenproblem handler
8605  delete Assembly_handler_pt;
8606  // Reset the assembly handler to the original handler
8607  Assembly_handler_pt = old_assembly_handler_pt;
8608  }
8609 
8610 
8611  //=======================================================================
8612  /// Stored the current values of the dofs
8613  //=======================================================================
8615  {
8616  // If memory has not been allocated, then allocated memory for the saved
8617  // dofs
8618  if (Saved_dof_pt == 0)
8619  {
8621  }
8622 
8623 #ifdef OOMPH_HAS_MPI
8624  // If the problem is distributed I have to do something different
8626  {
8627  // How many entries do we store locally?
8628  const unsigned n_row_local = Dof_distribution_pt->nrow_local();
8629 
8630  // Resize the vector
8631  Saved_dof_pt->resize(n_row_local);
8632 
8633  // Back 'em up
8634  for (unsigned i = 0; i < n_row_local; i++)
8635  {
8636  (*Saved_dof_pt)[i] = *(this->Dof_pt[i]);
8637  }
8638  }
8639  // Otherwise just store all the dofs
8640  else
8641 #endif
8642  {
8643  // Find the number of dofs
8644  unsigned long n_dof = ndof();
8645 
8646  // Resize the vector
8647  Saved_dof_pt->resize(n_dof);
8648 
8649  // Transfer the values over
8650  for (unsigned long n = 0; n < n_dof; n++)
8651  {
8652  (*Saved_dof_pt)[n] = dof(n);
8653  }
8654  }
8655  }
8656 
8657  //====================================================================
8658  /// Restore the saved dofs
8659  //====================================================================
8661  {
8662  // Check that we can do this
8663  if (Saved_dof_pt == 0)
8664  {
8665  throw OomphLibError(
8666  "There are no stored values, use store_current_dof_values()\n",
8667  OOMPH_CURRENT_FUNCTION,
8668  OOMPH_EXCEPTION_LOCATION);
8669  }
8670 
8671 
8672 #ifdef OOMPH_HAS_MPI
8673  // If the problem is distributed I have to do something different
8675  {
8676  // How many entries do we store locally?
8677  const unsigned n_row_local = Dof_distribution_pt->nrow_local();
8678 
8679  if (Saved_dof_pt->size() != n_row_local)
8680  {
8681  throw OomphLibError("The number of stored values is not equal to the "
8682  "current number of dofs\n",
8683  OOMPH_CURRENT_FUNCTION,
8684  OOMPH_EXCEPTION_LOCATION);
8685  }
8686 
8687  // Transfer the values over
8688  for (unsigned long n = 0; n < n_row_local; n++)
8689  {
8690  *(this->Dof_pt[n]) = (*Saved_dof_pt)[n];
8691  }
8692  }
8693  // Otherwise just restore all the dofs
8694  else
8695 #endif
8696  {
8697  // Find the number of dofs
8698  unsigned long n_dof = ndof();
8699 
8700  if (Saved_dof_pt->size() != n_dof)
8701  {
8702  throw OomphLibError("The number of stored values is not equal to the "
8703  "current number of dofs\n",
8704  OOMPH_CURRENT_FUNCTION,
8705  OOMPH_EXCEPTION_LOCATION);
8706  }
8707 
8708  // Transfer the values over
8709  for (unsigned long n = 0; n < n_dof; n++)
8710  {
8711  dof(n) = (*Saved_dof_pt)[n];
8712  }
8713  }
8714 
8715  // Delete the memory
8716  delete Saved_dof_pt;
8717  Saved_dof_pt = 0;
8718  }
8719 
8720  //======================================================================
8721  /// Assign the eigenvector passed to the function to the dofs
8722  //======================================================================
8724  {
8725  unsigned long n_dof = ndof();
8726  // Check that the eigenvector has the correct size
8727  if (eigenvector.nrow() != n_dof)
8728  {
8729  std::ostringstream error_message;
8730  error_message << "Eigenvector has size " << eigenvector.nrow()
8731  << ", not equal to the number of dofs in the problem,"
8732  << n_dof << std::endl;
8733 
8734  throw OomphLibError(
8735  error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
8736  }
8737 
8738  // Ensure that the eigenvector distribution matches the dof distribution
8739  // Copy vector
8740  DoubleVector eigenvector_dof = eigenvector;
8741  // Redistribute the copy to the dof distribution
8742  eigenvector_dof.redistribute(this->Dof_distribution_pt);
8743 
8744  // Loop over the dofs and assign the eigenvector
8745  for (unsigned long n = 0; n < eigenvector_dof.nrow_local(); n++)
8746  {
8747  dof(n) = eigenvector_dof[n];
8748  }
8749 // Of course we now need to synchronise
8750 #ifdef OOMPH_HAS_MPI
8751  this->synchronise_all_dofs();
8752 #endif
8753  }
8754 
8755 
8756  //======================================================================
8757  /// Add the eigenvector passed to the function to the dofs with
8758  /// magnitude epsilon
8759  //======================================================================
8760  void Problem::add_eigenvector_to_dofs(const double& epsilon,
8761  const DoubleVector& eigenvector)
8762  {
8763  unsigned long n_dof = ndof();
8764  // Check that the eigenvector has the correct size
8765  if (eigenvector.nrow() != n_dof)
8766  {
8767  std::ostringstream error_message;
8768  error_message << "Eigenvector has size " << eigenvector.nrow()
8769  << ", not equal to the number of dofs in the problem,"
8770  << n_dof << std::endl;
8771 
8772  throw OomphLibError(
8773  error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
8774  }
8775 
8776  // Ensure that the eigenvector distribution matches the dof distribution
8777  // Copy vector
8778  DoubleVector eigenvector_dof = eigenvector;
8779  // Redistribute the copy to the dof distribution
8780  eigenvector_dof.redistribute(this->Dof_distribution_pt);
8781 
8782 
8783  // Loop over the dofs and add the eigenvector
8784  // Only use local values
8785  for (unsigned long n = 0; n < eigenvector.nrow_local(); n++)
8786  {
8787  dof(n) += epsilon * eigenvector[n];
8788  }
8789 // Of course we now need to synchronise
8790 #ifdef OOMPH_HAS_MPI
8791  this->synchronise_all_dofs();
8792 #endif
8793  }
8794 
8795 
8796  //================================================================
8797  /// General Newton solver. Requires only a convergence tolerance.
8798  /// The linear solver takes a pointer to the problem (which defines
8799  /// the Jacobian \b J and the residual Vector \b r) and returns
8800  /// the solution \b x of the system
8801  /// \f[ {\bf J} {\bf x} = - \bf{r} \f].
8802  //================================================================
8804  {
8805  // Initialise timers
8806  double total_linear_solver_time = 0.0;
8807  double t_start = TimingHelpers::timer();
8808  Max_res.clear();
8809 
8810  // Find total number of dofs
8811  unsigned long n_dofs = ndof();
8812 
8813  // Set up the Vector to hold the solution
8814  DoubleVector dx;
8815 
8816  //-----Variables for the globally convergent Newton method------
8817 
8818  // Set up the vector to hold the gradient
8819  DoubleVector gradient;
8820 
8821  // Other variables
8822  double half_residual_squared = 0.0;
8823  double max_step = 0.0;
8824 
8825  //--------------------------------------------------------------
8826 
8827  // Set the counter
8828  unsigned count = 0;
8829  // Set the loop flag
8830  unsigned LOOP_FLAG = 1;
8831 
8833  {
8834 #ifdef OOMPH_HAS_MPI
8835  // Break if running in parallel
8837  {
8838  std::ostringstream error_stream;
8839  error_stream << "Globally convergent Newton method has not been "
8840  << "implemented in parallel yet!" << std::endl;
8841  throw OomphLibError(
8842  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
8843  }
8844 #endif
8845 
8846  // Get gradient
8848  // Reset the gradient (clear it), since the number of dofs and
8849  // hence the size of the DoubleVector might have changed
8851  }
8852 
8853  // Update anything that needs updating
8855 
8856  // Reset number of Newton iterations taken
8857  Nnewton_iter_taken = 0;
8858 
8859  // Now do the Newton loop
8860  do
8861  {
8862  count++;
8863 
8864  // Do any updates that are required
8866 
8867 
8868  // No degrees of freedom? What are you solving for?
8869  if (n_dofs == 0)
8870  {
8871  oomph_info << std::endl << std::endl << std::endl;
8872  oomph_info << "This is a bit bizarre: The problem has no dofs."
8873  << std::endl;
8874  oomph_info
8875  << "I'll just return from the Newton solver without doing anything."
8876  << std::endl;
8877 
8878  // Do any updates that would have been performed
8883 
8884  oomph_info << "I hope this is what you intended me to do..."
8885  << std::endl;
8886  oomph_info << std::endl
8887  << "Note: All actions_...() functions were called"
8888  << std::endl;
8889  oomph_info << std::endl << " before returning." << std::endl;
8890  oomph_info << std::endl << std::endl << std::endl;
8891  return;
8892  }
8893 
8894  // Calculate initial residuals
8895  if (count == 1)
8896  {
8897  // Is the problem nonlinear? If not ignore the pre-iteration
8898  // convergence check.
8900  {
8901 #ifdef OOMPH_HAS_MPI
8902  // Synchronise the solution on different processors (on each submesh)
8903  this->synchronise_all_dofs();
8904 #endif
8905 
8907  dx.clear();
8908  get_residuals(dx);
8909 
8910  // Get half of squared residual and find maximum step length
8911  // for step length control
8913  {
8914  half_residual_squared = 0.0;
8915  double sum = 0.0;
8916  for (unsigned i = 0; i < n_dofs; i++)
8917  {
8918  sum += (*Dof_pt[i]) * (*Dof_pt[i]);
8919  half_residual_squared += dx[i] * dx[i];
8920  }
8921  half_residual_squared *= 0.5;
8922  max_step = 100.0 * std::max(sqrt(sum), double(n_dofs));
8923  }
8924 
8925  // Get maximum residuals
8926  double maxres = dx.max();
8927  Max_res.push_back(maxres);
8928 
8930  {
8931  // Let's output the residuals
8932  // unsigned n_row_local = dx.distribution_pt()->nrow_local();
8933  // unsigned first_row = dx.distribution_pt()->first_row();
8934  // for(unsigned n=0;n<n_row_local;n++)
8935  //{
8936  // oomph_info << "residual: " << n + first_row << " " << dx[n] <<
8937  // "\n";
8938  //}
8939 
8940  oomph_info << "\nInitial Maximum residuals " << maxres << std::endl;
8941  }
8942 
8943  if ((maxres < Newton_solver_tolerance) &&
8945  {
8946  LOOP_FLAG = 0;
8947  continue;
8948  }
8949  }
8950  else
8951  {
8953  {
8954  oomph_info
8955  << "Linear problem -- convergence in one iteration assumed."
8956  << std::endl;
8957  }
8958  }
8959  }
8960 
8961 
8962  // Increment number of Newton iterations taken
8964 
8965  // Initialise timer for linear solver
8966  double t_solver_start = TimingHelpers::timer();
8967 
8968  // Now do the linear solve -- recycling Jacobian if requested
8970  {
8972  {
8973  oomph_info << "Not recomputing Jacobian! " << std::endl;
8974  }
8975 
8976  // If we're doing the first iteration and the problem is nonlinear,
8977  // the residuals have already been computed above during the
8978  // initial convergence check. Otherwise compute them here.
8979  if ((count != 1) || (!Problem_is_nonlinear)) get_residuals(dx);
8980 
8981  // Backup residuals
8982  DoubleVector resid(dx);
8983 
8984  // Resolve
8985  Linear_solver_pt->resolve(resid, dx);
8986  }
8987  else
8988  {
8990  {
8992  {
8993  oomph_info << "Enabling resolve" << std::endl;
8994  }
8996  }
8997  Linear_solver_pt->solve(this, dx);
8999  }
9000 
9001  // End of linear solver
9002  double t_solver_end = TimingHelpers::timer();
9003  total_linear_solver_time += t_solver_end - t_solver_start;
9004 
9006  {
9007  oomph_info << std::endl;
9008  oomph_info << "Time for linear solver (ndof=" << n_dofs << "): "
9010  t_solver_end - t_solver_start)
9011  << std::endl
9012  << std::endl;
9013  }
9014 
9015  // Subtract the new values from the true dofs
9017  double* dx_pt = dx.values_pt();
9018  unsigned ndof_local = Dof_distribution_pt->nrow_local();
9019 
9021  {
9022  // Get the gradient
9023  Linear_solver_pt->get_gradient(gradient);
9024 
9025  for (unsigned i = 0; i < ndof_local; i++)
9026  {
9027  dx_pt[i] *= -1.0;
9028  }
9029 
9030  // Update with steplength control
9031  Vector<double> unknowns_old(ndof_local);
9032 
9033  for (unsigned i = 0; i < ndof_local; i++)
9034  {
9035  unknowns_old[i] = *Dof_pt[i];
9036  }
9037 
9038  double half_residual_squared_old = half_residual_squared;
9039  globally_convergent_line_search(unknowns_old,
9040  half_residual_squared_old,
9041  gradient,
9042  dx,
9043  half_residual_squared,
9044  max_step);
9045  }
9046  // direct Newton update
9047  else
9048  {
9049  for (unsigned l = 0; l < ndof_local; l++)
9050  {
9051  *Dof_pt[l] -= Relaxation_factor * dx_pt[l];
9052  }
9053  }
9054 #ifdef OOMPH_HAS_MPI
9055  // Synchronise the solution on different processors (on each submesh)
9056  this->synchronise_all_dofs();
9057 #endif
9058 
9059  // Do any updates that are required
9062 
9063  // Maximum residuals
9064  double maxres = 0.0;
9065  // If the user has declared that the Problem is linear
9066  // we ignore the convergence check
9068  {
9069  // Get the maximum residuals
9070  // maxres = std::fabs(*std::max_element(dx.begin(),dx.end(),
9071  // AbsCmp<double>()));
9072  // oomph_info << "Maxres correction " << maxres << "\n";
9073 
9074  // Calculate the new residuals
9075  dx.clear();
9076  get_residuals(dx);
9077 
9078  // Get the maximum residuals
9079  maxres = dx.max();
9080  Max_res.push_back(maxres);
9081 
9083  {
9084  oomph_info << "Newton Step " << count << ": Maximum residuals "
9085  << maxres << std::endl
9086  << std::endl;
9087  }
9088  }
9089 
9090  // If we have converged jump straight to the test at the end of the loop
9091  if (maxres < Newton_solver_tolerance)
9092  {
9093  LOOP_FLAG = 0;
9094  continue;
9095  }
9096 
9097  // This section will not be reached if we have converged already
9098  // If the maximum number of residuals is too high or the maximum number
9099  // of iterations has been reached
9100  if ((maxres > Max_residuals) || (count == Max_newton_iterations))
9101  {
9102  // Print a warning -- regardless of what the throw does
9103  if (maxres > Max_residuals)
9104  {
9105  oomph_info << "Max. residual (" << Max_residuals
9106  << ") has been exceeded in Newton solver." << std::endl;
9107  }
9108  if (count == Max_newton_iterations)
9109  {
9110  oomph_info << "Reached max. number of iterations ("
9111  << Max_newton_iterations << ") in Newton solver."
9112  << std::endl;
9113  }
9114  // Now throw...
9115  throw NewtonSolverError(count, maxres);
9116  }
9117 
9118  } while (LOOP_FLAG);
9119 
9120  // Now update anything that needs updating
9122 
9123  // Finalise/doc timings
9125  {
9126  oomph_info << std::endl;
9127  oomph_info << "Total time for linear solver (ndof=" << n_dofs << "): "
9129  total_linear_solver_time)
9130  << std::endl;
9131  }
9132 
9133  double t_end = TimingHelpers::timer();
9134  double total_time = t_end - t_start;
9135 
9137  {
9138  oomph_info << "Total time for Newton solver (ndof=" << n_dofs << "): "
9140  << std::endl;
9141  }
9142  if (total_time > 0.0)
9143  {
9145  {
9146  oomph_info << "Time outside linear solver : "
9147  << (total_time - total_linear_solver_time) / total_time *
9148  100.0
9149  << " %" << std::endl;
9150  }
9151  }
9152  else
9153  {
9155  {
9156  oomph_info << "Time outside linear solver : "
9157  << "[too fast]" << std::endl;
9158  }
9159  }
9160  if (!Shut_up_in_newton_solve) oomph_info << std::endl;
9161  }
9162 
9163  //========================================================================
9164  /// Helper function for the globally convergent Newton solver
9165  //========================================================================
9167  const Vector<double>& x_old,
9168  const double& half_residual_squared_old,
9169  DoubleVector& gradient,
9170  DoubleVector& newton_dir,
9171  double& half_residual_squared,
9172  const double& stpmax)
9173  {
9174  const double min_fct_decrease = 1.0e-4;
9175  double convergence_tol_on_x = 1.0e-16;
9176  double f_aux = 0.0;
9177  double lambda_aux = 0.0;
9178  double proposed_lambda;
9179  unsigned long n_dof = ndof();
9180  double sum = 0.0;
9181  for (unsigned i = 0; i < n_dof; i++)
9182  {
9183  sum += newton_dir[i] * newton_dir[i];
9184  }
9185  sum = sqrt(sum);
9186  if (sum > stpmax)
9187  {
9188  for (unsigned i = 0; i < n_dof; i++)
9189  {
9190  newton_dir[i] *= stpmax / sum;
9191  }
9192  }
9193  double slope = 0.0;
9194  for (unsigned i = 0; i < n_dof; i++)
9195  {
9196  slope += gradient[i] * newton_dir[i];
9197  }
9198  if (slope >= 0.0)
9199  {
9200  std::ostringstream warn_message;
9201  warn_message << "WARNING: Non-negative slope, probably due to a "
9202  << " roundoff \nproblem in the linesearch: slope=" << slope
9203  << "\n";
9204  OomphLibWarning(warn_message.str(),
9205  "Problem::globally_convergent_line_search()",
9206  OOMPH_EXCEPTION_LOCATION);
9207  }
9208  double test = 0.0;
9209  for (unsigned i = 0; i < n_dof; i++)
9210  {
9211  double temp =
9212  std::fabs(newton_dir[i]) / std::max(std::fabs(x_old[i]), 1.0);
9213  if (temp > test) test = temp;
9214  }
9215  double lambda_min = convergence_tol_on_x / test;
9216  double lambda = 1.0;
9217  while (true)
9218  {
9219  for (unsigned i = 0; i < n_dof; i++)
9220  {
9221  *Dof_pt[i] = x_old[i] + lambda * newton_dir[i];
9222  }
9223 
9224  // Evaluate current residuals
9225  DoubleVector residuals;
9226  get_residuals(residuals);
9227  half_residual_squared = 0.0;
9228  for (unsigned i = 0; i < n_dof; i++)
9229  {
9230  half_residual_squared += residuals[i] * residuals[i];
9231  }
9232  half_residual_squared *= 0.5;
9233 
9234  if (lambda < lambda_min)
9235  {
9236  for (unsigned i = 0; i < n_dof; i++) *Dof_pt[i] = x_old[i];
9237 
9238  std::ostringstream warn_message;
9239  warn_message << "WARNING: Line search converged on x only!\n";
9240  OomphLibWarning(warn_message.str(),
9241  "Problem::globally_convergent_line_search()",
9242  OOMPH_EXCEPTION_LOCATION);
9243  return;
9244  }
9245  else if (half_residual_squared <=
9246  half_residual_squared_old + min_fct_decrease * lambda * slope)
9247  {
9248  oomph_info << "Returning from linesearch with lambda=" << lambda
9249  << std::endl;
9250  return;
9251  }
9252  else
9253  {
9254  if (lambda == 1.0)
9255  {
9256  proposed_lambda =
9257  -slope /
9258  (2.0 * (half_residual_squared - half_residual_squared_old - slope));
9259  }
9260  else
9261  {
9262  double r1 =
9263  half_residual_squared - half_residual_squared_old - lambda * slope;
9264  double r2 = f_aux - half_residual_squared_old - lambda_aux * slope;
9265  double a_poly =
9266  (r1 / (lambda * lambda) - r2 / (lambda_aux * lambda_aux)) /
9267  (lambda - lambda_aux);
9268  double b_poly = (-lambda_aux * r1 / (lambda * lambda) +
9269  lambda * r2 / (lambda_aux * lambda_aux)) /
9270  (lambda - lambda_aux);
9271  if (a_poly == 0.0)
9272  {
9273  proposed_lambda = -slope / (2.0 * b_poly);
9274  }
9275  else
9276  {
9277  double discriminant = b_poly * b_poly - 3.0 * a_poly * slope;
9278  if (discriminant < 0.0)
9279  {
9280  proposed_lambda = 0.5 * lambda;
9281  }
9282  else if (b_poly <= 0.0)
9283  {
9284  proposed_lambda = (-b_poly + sqrt(discriminant)) / (3.0 * a_poly);
9285  }
9286  else
9287  {
9288  proposed_lambda = -slope / (b_poly + sqrt(discriminant));
9289  }
9290  }
9291  if (proposed_lambda > 0.5 * lambda)
9292  {
9293  proposed_lambda = 0.5 * lambda;
9294  }
9295  }
9296  }
9297  lambda_aux = lambda;
9298  f_aux = half_residual_squared;
9299  lambda = std::max(proposed_lambda, 0.1 * lambda);
9300  }
9301  }
9302 
9303 
9304  //========================================================================
9305  /// Solve a steady problem, in the context of an overall unsteady problem.
9306  /// This is achieved by setting the weights in the timesteppers to be zero
9307  /// which has the effect of rendering them steady timesteppers
9308  /// The optional argument max_adapt specifies the max. number of
9309  /// adaptations of all refineable submeshes are performed to
9310  /// achieve the the error targets specified in the refineable submeshes.
9311  //========================================================================
9312  void Problem::steady_newton_solve(unsigned const& max_adapt)
9313  {
9314  // Find out how many timesteppers there are
9315  unsigned n_time_steppers = ntime_stepper();
9316 
9317  // Vector of bools to store the is_steady status of the various
9318  // timesteppers when we came in here
9319  std::vector<bool> was_steady(n_time_steppers);
9320 
9321  // Loop over them all and make them (temporarily) static
9322  for (unsigned i = 0; i < n_time_steppers; i++)
9323  {
9324  was_steady[i] = time_stepper_pt(i)->is_steady();
9326  }
9327 
9328  try
9329  {
9330  // Solve the non-linear problem with Newton's method
9331  if (max_adapt == 0)
9332  {
9333  newton_solve();
9334  }
9335  else
9336  {
9337  newton_solve(max_adapt);
9338  }
9339  }
9340  // Catch any exceptions thrown in the Newton solver
9341  catch (NewtonSolverError& error)
9342  {
9343  oomph_info << std::endl
9344  << "USER-DEFINED ERROR IN NEWTON SOLVER " << std::endl;
9345  // Check whether it's the linear solver
9346  if (error.linear_solver_error())
9347  {
9348  oomph_info << "ERROR IN THE LINEAR SOLVER" << std::endl;
9349  }
9350  // Check to see whether we have reached Max_iterations
9351  else if (error.iterations() == Max_newton_iterations)
9352  {
9353  oomph_info << "MAXIMUM NUMBER OF ITERATIONS (" << error.iterations()
9354  << ") REACHED WITHOUT CONVERGENCE " << std::endl;
9355  }
9356  // If not, it must be that we have exceeded the maximum residuals
9357  else
9358  {
9359  oomph_info << "MAXIMUM RESIDUALS: " << error.maxres()
9360  << " EXCEEDS PREDEFINED MAXIMUM " << Max_residuals
9361  << std::endl;
9362  }
9363 
9364  // Die horribly!!
9365  std::ostringstream error_stream;
9366  error_stream << "Error occured in Newton solver. " << std::endl;
9367  throw OomphLibError(
9368  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
9369  }
9370 
9371 
9372  // Reset the is_steady status of all timesteppers that
9373  // weren't already steady when we came in here and reset their
9374  // weights
9375  for (unsigned i = 0; i < n_time_steppers; i++)
9376  {
9377  if (!was_steady[i])
9378  {
9380  }
9381  }
9382 
9383  // Since we performed a steady solve, the history values
9384  // now have to be set as if we had performed an impulsive start from
9385  // the current solution. This ensures that the time-derivatives
9386  // evaluate to zero even now that the timesteppers have been
9387  // reactivated.
9389  }
9390 
9391  //===========================================================================
9392  /// Perform a basic continuation step using Newton's method. The governing
9393  /// parameter of the problem is passed as a pointer to the routine. The
9394  /// number of Newton steps taken is returned
9395  //==========================================================================
9396  unsigned Problem::newton_solve_continuation(double* const& parameter_pt)
9397  {
9398  // Set up memory for z
9399  // unsigned long n_dofs = ndof();
9400  // LinearAlgebraDistribution dist(Communicator_pt,n_dofs,false);
9401  // DoubleVector z(&dist,0.0);
9402  DoubleVector z;
9403  // Call the solver
9404  return newton_solve_continuation(parameter_pt, z);
9405  }
9406 
9407 
9408  //===================================================================
9409  /// This function performs a basic continuation step using the Newton method.
9410  /// The number of Newton steps taken is returned, to be used in any
9411  /// external step-size control routines.
9412  /// The governing parameter of the problem is passed as a pointer to the
9413  /// routine, as is the sign of the Jacobian and a Vector in which
9414  /// to store the derivatives wrt the parameter, if required.
9415  //==================================================================
9416  unsigned Problem::newton_solve_continuation(double* const& parameter_pt,
9417  DoubleVector& z)
9418  {
9419  // Find the total number of dofs
9420  // unsigned long n_dofs = ndof();
9421 
9422  // Find the local number of dofs
9423  unsigned ndof_local = Dof_distribution_pt->nrow_local();
9424 
9425  // create the distribution (not distributed)
9426  // LinearAlgebraDistribution dist(this->communicator_pt(),n_dofs,false);
9427 
9428  // Assign memory for solutions of the equations
9429  // DoubleVector y(&dist,0.0);
9430  DoubleVector y;
9431 
9432  // Assign memory for the dot products of the uderivatives and y and z
9433  double uderiv_dot_y = 0.0, uderiv_dot_z = 0.0;
9434  // Set and initialise the counter
9435  unsigned count = 0;
9436  // Set the loop flag
9437  unsigned LOOP_FLAG = 1;
9438 
9439  // Update anything that needs updating
9441 
9442  // Check the arc-length constraint
9443  double arc_length_constraint_residual = 0.0;
9444 
9445  // Are we storing the matrix in the linear solve
9446  bool enable_resolve = Linear_solver_pt->is_resolve_enabled();
9447 
9448  // For this problem, we must store the residuals
9450 
9451  // Now do the Newton loop
9452  do
9453  {
9454  count++;
9455 
9456  // Do any updates that are required
9458 
9459  // Calculate initial residuals
9460  if (count == 1)
9461  {
9462 #ifdef OOMPH_HAS_MPI
9463  // Synchronise the solution on different processors (on each submesh)
9464  this->synchronise_all_dofs();
9465 #endif
9466 
9468  y.clear();
9469  get_residuals(y);
9470  // Get maximum residuals, using our own abscmp function
9471  double maxres = y.max();
9472 
9473  // Assemble the residuals for the arc-length step
9474  arc_length_constraint_residual = 0.0;
9475  // Add the variables
9476  for (unsigned long l = 0; l < ndof_local; l++)
9477  {
9478  arc_length_constraint_residual +=
9479  dof_derivative(l) * (*Dof_pt[l] - dof_current(l));
9480  }
9481 
9482  // Now reduce if we have been distributed
9483 #ifdef OOMPH_HAS_MPI
9484  double arc_length_cons_res2 = arc_length_constraint_residual;
9485  if ((Dof_distribution_pt->distributed()) &&
9486  (Dof_distribution_pt->communicator_pt()->nproc() > 1))
9487  {
9488  MPI_Allreduce(&arc_length_constraint_residual,
9489  &arc_length_cons_res2,
9490  1,
9491  MPI_DOUBLE,
9492  MPI_SUM,
9493  Dof_distribution_pt->communicator_pt()->mpi_comm());
9494  }
9495  arc_length_constraint_residual = arc_length_cons_res2;
9496 #endif
9497 
9498  arc_length_constraint_residual *= Theta_squared;
9499  arc_length_constraint_residual +=
9500  Parameter_derivative * (*parameter_pt - Parameter_current) -
9501  Ds_current;
9502 
9503  // Is it the max
9504  if (std::fabs(arc_length_constraint_residual) > maxres)
9505  {
9506  maxres = std::fabs(arc_length_constraint_residual);
9507  }
9508 
9509  // Find the max
9511  {
9512  oomph_info << "Initial Maximum residuals " << maxres << std::endl;
9513  }
9514 
9515  // If we are below the Tolerance, then return immediately
9516  if ((maxres < Newton_solver_tolerance) &&
9518  {
9519  LOOP_FLAG = 0;
9520  count = 0;
9521  continue;
9522  }
9523  }
9524 
9525  // If it's the block hopf solver we need to solve for both rhs's
9526  // simultaneously. This is because the block decomposition involves
9527  // solves with two different matrices and storing both at once to
9528  // allow general resolves would be more expensive than necessary.
9529  if (dynamic_cast<BlockHopfLinearSolver*>(Linear_solver_pt))
9530  {
9531  // Get the vector dresiduals/dparameter
9532  z.clear();
9533  get_derivative_wrt_global_parameter(parameter_pt, z);
9534 
9535  // Copy rhs vector into local storage so it doesn't get overwritten
9536  // if the linear solver decides to initialise the solution vector, say,
9537  // which it's quite entitled to do!
9538  DoubleVector input_z(z);
9539 
9540  // Solve the system for the two right-hand sides.
9541  dynamic_cast<BlockHopfLinearSolver*>(Linear_solver_pt)
9542  ->solve_for_two_rhs(this, y, input_z, z);
9543  }
9544  // Otherwise
9545  else
9546  {
9547  // Solve the standard problem
9548  Linear_solver_pt->solve(this, y);
9549 
9550  // Get the vector dresiduals/dparameter
9551  z.clear();
9552  get_derivative_wrt_global_parameter(parameter_pt, z);
9553 
9554  // Copy rhs vector into local storage so it doesn't get overwritten
9555  // if the linear solver decides to initialise the solution vector, say,
9556  // which it's quite entitled to do!
9557  DoubleVector input_z(z);
9558 
9559  // Redistribute the RHS to match the linear solver
9560  // input_z.redistribute(Linear_solver_pt->distribution_pt());
9561  // Do not clear z because we assume that it has dR/dparam
9562  z.clear();
9563  // Now resolve the system with the new RHS
9564  Linear_solver_pt->resolve(input_z, z);
9565  }
9566 
9567  // Redistribute the results into the natural distribution
9570 
9571  // Now we need to calculate dparam, for which we must calculate the
9572  // dot product of the derivatives and y and z
9573  // Reset these values to zero
9574  uderiv_dot_y = 0.0;
9575  uderiv_dot_z = 0.0;
9576  // Now calculate the dot products of the derivative and the solutions
9577  // of the linear system
9578  // Cache pointers to the data in the distributed vectors
9579  double* const y_pt = y.values_pt();
9580  double* const z_pt = z.values_pt();
9581  for (unsigned long l = 0; l < ndof_local; l++)
9582  {
9583  uderiv_dot_y += dof_derivative(l) * y_pt[l];
9584  uderiv_dot_z += dof_derivative(l) * z_pt[l];
9585  }
9586 
9587  // Now reduce if we have been distributed
9588 #ifdef OOMPH_HAS_MPI
9589  // Create send and receive arrays of size two
9590  double uderiv_dot[2];
9591  double uderiv_dot2[2];
9592  uderiv_dot[0] = uderiv_dot_y;
9593  uderiv_dot[1] = uderiv_dot_z;
9594  uderiv_dot2[0] = uderiv_dot_y;
9595  uderiv_dot2[1] = uderiv_dot_z;
9596  // Now reduce both together
9597  if ((Dof_distribution_pt->distributed()) &&
9598  (Dof_distribution_pt->communicator_pt()->nproc() > 1))
9599  {
9600  MPI_Allreduce(uderiv_dot,
9601  uderiv_dot2,
9602  2,
9603  MPI_DOUBLE,
9604  MPI_SUM,
9605  Dof_distribution_pt->communicator_pt()->mpi_comm());
9606  }
9607  uderiv_dot_y = uderiv_dot2[0];
9608  uderiv_dot_z = uderiv_dot2[1];
9609 #endif
9610 
9611  // Now scale the results
9612  uderiv_dot_y *= Theta_squared;
9613  uderiv_dot_z *= Theta_squared;
9614 
9615  // The set the change in the parameter, given by the pseudo-arclength
9616  // equation. Note that here we are assuming that the arc-length
9617  // equation is always exactly zero,
9618  // which seems to work OK, and saves on some storage.
9619  // In fact, it's more subtle than that. If we include this
9620  // proper residual then we will have to solve the eigenproblem.
9621  // This will make the solver more robust and *should* be done
9622  // ... at some point.
9623  double dparam = (arc_length_constraint_residual - uderiv_dot_y) /
9624  (Parameter_derivative - uderiv_dot_z);
9625 
9626  // Set the new value of the parameter
9627  *parameter_pt -= dparam;
9628 
9629  // Update the values of the other degrees of freedom
9630  for (unsigned long l = 0; l < ndof_local; l++)
9631  {
9632  *Dof_pt[l] -= y_pt[l] - dparam * z_pt[l];
9633  }
9634 
9635  // Calculate the new residuals
9636 #ifdef OOMPH_HAS_MPI
9637  // Synchronise the solution on different processors (on each submesh)
9638  this->synchronise_all_dofs();
9639 #endif
9640 
9641  // Do any updates that are required
9644 
9645  y.clear();
9646  get_residuals(y);
9647 
9648  // Get the maximum residuals
9649  double maxres = y.max();
9650 
9651  // Assemble the residuals for the arc-length step
9652  arc_length_constraint_residual = 0.0;
9653  // Add the variables
9654  for (unsigned long l = 0; l < ndof_local; l++)
9655  {
9656  arc_length_constraint_residual +=
9657  dof_derivative(l) * (*Dof_pt[l] - dof_current(l));
9658  }
9659 
9660  // Now reduce if we have been distributed
9661 #ifdef OOMPH_HAS_MPI
9662  double arc_length_cons_res2 = arc_length_constraint_residual;
9663  if ((Dof_distribution_pt->distributed()) &&
9664  (Dof_distribution_pt->communicator_pt()->nproc() > 1))
9665  {
9666  MPI_Allreduce(&arc_length_constraint_residual,
9667  &arc_length_cons_res2,
9668  1,
9669  MPI_DOUBLE,
9670  MPI_SUM,
9671  Dof_distribution_pt->communicator_pt()->mpi_comm());
9672  }
9673  arc_length_constraint_residual = arc_length_cons_res2;
9674 #endif
9675 
9676  arc_length_constraint_residual *= Theta_squared;
9677  arc_length_constraint_residual +=
9678  Parameter_derivative * (*parameter_pt - Parameter_current) - Ds_current;
9679 
9680  // Is it the max
9681  if (std::fabs(arc_length_constraint_residual) > maxres)
9682  {
9683  maxres = std::fabs(arc_length_constraint_residual);
9684  }
9685 
9687  {
9688  oomph_info << "Continuation Step " << count << ": Maximum residuals "
9689  << maxres << "\n";
9690  }
9691 
9692  // If we have converged jump straight to the test at the end of the loop
9693  if (maxres < Newton_solver_tolerance)
9694  {
9695  LOOP_FLAG = 0;
9696  continue;
9697  }
9698 
9699  // This section will not be reached if we have converged already
9700  // If the maximum number of residuals is too high or the maximum number
9701  // of iterations has been reached
9702  if ((maxres > Max_residuals) || (count == Max_newton_iterations))
9703  {
9704  throw NewtonSolverError(count, maxres);
9705  }
9706 
9707  } while (LOOP_FLAG);
9708 
9709  // Now update anything that needs updating
9711 
9712  // Reset the storage of the matrix on the linear solver to what it was
9713  // on entry to this routine
9714  if (enable_resolve)
9715  {
9717  }
9718  else
9719  {
9721  }
9722 
9723  // Return the number of Newton Steps taken
9724  return count;
9725  }
9726 
9727  //=========================================================================
9728  /// A function to calculate the derivatives wrt the arc-length. This version
9729  /// of the function actually does a linear solve so that the derivatives
9730  /// are calculated "exactly" rather than using the values at the Newton
9731  /// step just before convergence. This is only necessary in spatially adaptive
9732  /// problems, in which the number of degrees of freedom changes and so
9733  /// the appropriate derivatives must be calculated for the new variables.
9734  //=========================================================================
9735  void Problem::calculate_continuation_derivatives(double* const& parameter_pt)
9736  {
9737  // Find the number of degrees of freedom in the problem
9738  const unsigned long n_dofs = ndof();
9739 
9740  // create a non-distributed z vector
9741  LinearAlgebraDistribution dist(Communicator_pt, n_dofs, false);
9742 
9743  // Assign memory for solutions of the equations
9744  DoubleVector z(&dist, 0.0);
9745 
9746  // If it's the block hopf solver need to solve for both RHS
9747  // at once, but this would all be alleviated if we have the solve
9748  // for the non-residuals RHS.
9749  if (dynamic_cast<BlockHopfLinearSolver*>(Linear_solver_pt))
9750  {
9751  // Get the vector dresiduals/dparameter
9752  get_derivative_wrt_global_parameter(parameter_pt, z);
9753 
9754  // Copy rhs vector into local storage so it doesn't get overwritten
9755  // if the linear solver decides to initialise the solution vector, say,
9756  // which it's quite entitled to do!
9757  DoubleVector dummy(&dist, 0.0), input_z(z);
9758 
9759  // Solve for the two RHSs
9760  dynamic_cast<BlockHopfLinearSolver*>(Linear_solver_pt)
9761  ->solve_for_two_rhs(this, dummy, input_z, z);
9762  }
9763  // Otherwise we can use the normal resolve
9764  else
9765  {
9766  // Save the status before entry to this routine
9767  bool enable_resolve = Linear_solver_pt->is_resolve_enabled();
9768 
9769  // We need to do resolves
9771 
9772  // Solve the standard problem, we only want to make sure that
9773  // we factorise the matrix, if it has not been factorised. We shall
9774  // ignore the return value of z.
9775  Linear_solver_pt->solve(this, z);
9776 
9777  // Get the vector dresiduals/dparameter
9778  get_derivative_wrt_global_parameter(parameter_pt, z);
9779 
9780 
9781  // Copy rhs vector into local storage so it doesn't get overwritten
9782  // if the linear solver decides to initialise the solution vector, say,
9783  // which it's quite entitled to do!
9784  DoubleVector input_z(z);
9785 
9786  // Now resolve the system with the new RHS and overwrite the solution
9787  Linear_solver_pt->resolve(input_z, z);
9788 
9789  // Restore the storage status of the linear solver
9790  if (enable_resolve)
9791  {
9793  }
9794  else
9795  {
9797  }
9798  }
9799 
9800  // Now, we can calculate the derivatives, etc
9802  }
9803 
9804  //=======================================================================
9805  /// A function to calculate the derivatives with respect to the arc-length
9806  /// required for continuation. The arguments is the solution of the
9807  /// linear system,
9808  /// Jz = dR/dparameter, that gives du/dparameter and the direction
9809  /// output from the newton_solve_continuation function. The derivatives
9810  /// are stored in the ContinuationParameters namespace.
9811  //===================================================================
9813  {
9814  // Calculate the continuation derivatives
9816 
9817  // Scale the value of theta if the control flag is set
9818  if (Scale_arc_length)
9819  {
9820  // Don't divide by zero!
9821  if (Parameter_derivative != 1.0)
9822  {
9827 
9828  // Recalculate the continuation derivatives with the new scaled values
9830  }
9831  }
9832  }
9833 
9834  //=======================================================================
9835  /// A function to calculate the derivatives with respect to the arc-length
9836  /// required for continuation using finite differences.
9837  //===================================================================
9839  double* const& parameter_pt)
9840  {
9841  // Calculate the continuation derivatives
9843 
9844  // Scale the value of theta if the control flag is set
9845  if (Scale_arc_length)
9846  {
9847  // Don't divide by zero!
9848  if (Parameter_derivative != 1.0)
9849  {
9854 
9855  // Recalculate the continuation derivatives with the new scaled values
9857  }
9858  }
9859  }
9860 
9861  //======================================================================
9862  /// Function that returns a boolean flag to indicate whether the pointer
9863  /// parameter_pt refers to memory that is a value in a Data object used
9864  /// within the problem
9865  //======================================================================
9867  double* const& parameter_pt)
9868  {
9869  // Firstly check the global data
9870  const unsigned n_global = Global_data_pt.size();
9871  for (unsigned i = 0; i < n_global; ++i)
9872  {
9873  // If we find it then return true
9874  if (Global_data_pt[i]->does_pointer_correspond_to_value(parameter_pt))
9875  {
9876  return true;
9877  }
9878  }
9879 
9880  // If we find the pointer in the mesh data return true
9881  if (Mesh_pt->does_pointer_correspond_to_mesh_data(parameter_pt))
9882  {
9883  return true;
9884  }
9885 
9886  // Loop over the submeshes to handle the case of spine data
9887  const unsigned n_sub_mesh = this->nsub_mesh();
9888  // If there is only one mesh
9889  if (n_sub_mesh == 0)
9890  {
9891  if (SpineMesh* const spine_mesh_pt = dynamic_cast<SpineMesh*>(Mesh_pt))
9892  {
9893  if (spine_mesh_pt->does_pointer_correspond_to_spine_data(parameter_pt))
9894  {
9895  return true;
9896  }
9897  }
9898  }
9899  // Otherwise loop over the sub meshes
9900  else
9901  {
9902  // Assign global equation numbers first
9903  for (unsigned i = 0; i < n_sub_mesh; i++)
9904  {
9905  if (SpineMesh* const spine_mesh_pt =
9906  dynamic_cast<SpineMesh*>(Sub_mesh_pt[i]))
9907  {
9908  if (spine_mesh_pt->does_pointer_correspond_to_spine_data(
9909  parameter_pt))
9910  {
9911  return true;
9912  }
9913  }
9914  }
9915  }
9916 
9917  // If we have got here then the data is not stored in the problem, so return
9918  // false
9919  return false;
9920  }
9921 
9922 
9923  //=======================================================================
9924  /// A private helper function to
9925  /// calculate the derivatives with respect to the arc-length
9926  /// required for continuation. The arguments is the solution of the
9927  /// linear system,
9928  /// Jz = dR/dparameter, that gives du/dparameter and the direction
9929  /// output from the newton_solve_continuation function. The derivatives
9930  /// are stored in the ContinuationParameters namespace.
9931  //===================================================================
9933  {
9934  // Find the number of degrees of freedom in the problem
9935  // unsigned long n_dofs = ndof();
9936  // Find the number of local dofs in the problem
9937  const unsigned long ndof_local = Dof_distribution_pt->nrow_local();
9938 
9939  // Work out the continuation direction
9940  // The idea is that (du/ds)_{old} . (du/ds)_{new} >= 0
9941  // if the direction is to remain the same.
9942  // du/ds_{new} = [dlambda/ds; du/ds] = [dlambda/ds ; - dlambda/ds z]
9943  // so (du/ds)_{new} . (du/ds)_{old}
9944  // = dlambda/ds [1 ; - z] . [ Parameter_derivative ; Dof_derivatives]
9945  // = dlambda/ds (Parameter_derivative - Dof_derivative . z)
9946 
9947  // Create a local copy of z that can be redistributed without breaking
9948  // the constness of z
9949  DoubleVector local_z(z);
9950 
9951  // Redistribute z so that it has the (natural) dof distribution
9953 
9954  // Calculate the local contribution to the Continuation direction
9955  Continuation_direction = 0.0;
9956  // Cache the pointer to z
9957  double* const local_z_pt = local_z.values_pt();
9958  for (unsigned long l = 0; l < ndof_local; l++)
9959  {
9960  Continuation_direction -= dof_derivative(l) * local_z_pt[l];
9961  }
9962 
9963  // Now reduce if we have been distributed
9964 #ifdef OOMPH_HAS_MPI
9965  double cont_dir2 = Continuation_direction;
9966  if ((Dof_distribution_pt->distributed()) &&
9967  (Dof_distribution_pt->communicator_pt()->nproc() > 1))
9968  {
9969  MPI_Allreduce(&Continuation_direction,
9970  &cont_dir2,
9971  1,
9972  MPI_DOUBLE,
9973  MPI_SUM,
9974  Dof_distribution_pt->communicator_pt()->mpi_comm());
9975  }
9976  Continuation_direction = cont_dir2;
9977 #endif
9978 
9979  // Add parameter derivative
9981 
9982  // Calculate the magnitude of the du/ds Vector
9983 
9984  // Note that actually, we are usually approximating by using the value at
9985  // newton step just before convergence, which saves one additional
9986  // Newton solve.
9987 
9988  // First calculate the magnitude of du/dparameter, chi
9989  double chi = local_z.dot(local_z);
9990 
9991  // Calculate the current derivative of the parameter wrt the arc-length
9992  Parameter_derivative = 1.0 / sqrt(1.0 + Theta_squared * chi);
9993 
9994  // If the dot product of the current derivative wrt the Direction
9995  // is less than zero, switch the sign of the derivative to ensure
9996  // smooth continuation
9998  {
9999  Parameter_derivative *= -1.0;
10000  }
10001 
10002  // Resize the derivatives array, if necessary
10004  {
10005  if (Dof_derivative.size() != ndof_local)
10006  {
10007  Dof_derivative.resize(ndof_local, 0.0);
10008  }
10009  }
10010  // Calculate the new derivatives wrt the arc-length
10011  for (unsigned long l = 0; l < ndof_local; l++)
10012  {
10013  // This comes from the formulation J u_dot + dr/dlambda lambda_dot = 0
10014  // on the curve and then it follows that.
10015  dof_derivative(l) = -Parameter_derivative * local_z_pt[l];
10016  }
10017  }
10018 
10019  //=======================================================================
10020  /// A private helper function to
10021  /// calculate the derivatives with respect to the arc-length
10022  /// required for continuation using finite differences.
10023  //===================================================================
10025  double* const& parameter_pt)
10026  {
10027  // Find the number of values
10028  // const unsigned long n_dofs = this->ndof();
10029  // Find the number of local dofs in the problem
10030  const unsigned long ndof_local = Dof_distribution_pt->nrow_local();
10031 
10032  // Temporary storage for the finite-difference approximation to the helper
10033  Vector<double> z(ndof_local);
10034  double length = 0.0;
10035  // Calculate the change in values and contribution to total length
10036  for (unsigned long l = 0; l < ndof_local; l++)
10037  {
10038  z[l] = (*Dof_pt[l] - Dof_current[l]) / Ds_current;
10039  length += Theta_squared * z[l] * z[l];
10040  }
10041 
10042  // Reduce if parallel
10043 #ifdef OOMPH_HAS_MPI
10044  double length2 = length;
10045  if ((Dof_distribution_pt->distributed()) &&
10046  (Dof_distribution_pt->communicator_pt()->nproc() > 1))
10047  {
10048  MPI_Allreduce(&length,
10049  &length2,
10050  1,
10051  MPI_DOUBLE,
10052  MPI_SUM,
10053  Dof_distribution_pt->communicator_pt()->mpi_comm());
10054  }
10055  length = length2;
10056 #endif
10057 
10058  // Calculate change in parameter
10059  double Z = (*parameter_pt - Parameter_current) / Ds_current;
10060  length += Z * Z;
10061 
10062  // Scale the approximations to the derivatives
10063  length = sqrt(length);
10064  for (unsigned long l = 0; l < ndof_local; l++)
10065  {
10066  dof_derivative(l) = z[l] / length;
10067  }
10068  Parameter_derivative = Z / length;
10069  }
10070 
10071 
10072  /// Virtual function that is used to symmetrise the problem so that
10073  /// the current solution exactly satisfies any symmetries within the system.
10074  /// Used when adpativly solving pitchfork detection problems when small
10075  /// asymmetries in the coarse solution can be magnified
10076  /// leading to very inaccurate answers on the fine mesh.
10077  /// This is always problem-specific and must be filled in by the user
10078  /// The default issues a warning
10080  {
10081  std::ostringstream warn_message;
10082  warn_message
10083  << "Warning: This function is called after spatially adapting the\n"
10084  << "eigenfunction associated with a pitchfork bifurcation and should\n"
10085  << "ensure that the exact (anti-)symmetries of problem are enforced\n"
10086  << "within that eigenfunction. It is problem specific and must be\n"
10087  << "filled in by the user if required.\n"
10088  << "A sign of problems is if the slack paramter gets too large and\n"
10089  << "if the solution at the Pitchfork is not symmetric.\n";
10091  warn_message.str(),
10092  "Problem::symmetrise_eigenfunction_for_adaptive_pitchfork_tracking()",
10093  OOMPH_EXCEPTION_LOCATION);
10094  }
10095 
10096  //====================================================================
10097  /// Return pointer to the parameter that is used in the
10098  /// bifurcation detection. If we are not tracking a bifurcation then
10099  /// an error will be thrown by the AssemblyHandler
10100  //====================================================================
10102  {
10104  }
10105 
10106  //====================================================================
10107  /// Return the eigenfunction calculated as part of a
10108  /// bifurcation tracking process. If we are not tracking a bifurcation
10109  /// then an error will be thrown by the AssemblyHandler
10110  //======================================================================
10112  Vector<DoubleVector>& eigenfunction)
10113  {
10114  // Simply call the appropriate assembly handler function
10115  Assembly_handler_pt->get_eigenfunction(eigenfunction);
10116  }
10117 
10118  //============================================================
10119  /// Activate the fold tracking system by changing the assembly
10120  /// handler and initialising it using the parameter addressed
10121  /// by parameter_pt.
10122  //============================================================
10123  void Problem::activate_fold_tracking(double* const& parameter_pt,
10124  const bool& block_solve)
10125  {
10126  // Reset the assembly handler to default
10128  // Set the new assembly handler. Note that the constructor actually
10129  // solves the original problem to get some initial conditions, but
10130  // this is OK because the RHS is always evaluated before assignment.
10131  Assembly_handler_pt = new FoldHandler(this, parameter_pt);
10132 
10133  // If we are using a block solver, we must set the linear solver pointer
10134  // to the block fold solver. The present linear solver is
10135  // used by the block solver and so must be passed as an argument.
10136  // The destructor of the Fold handler returns the linear
10137  // solver to the original non-block version.
10138  if (block_solve)
10139  {
10141  }
10142  }
10143 
10144  //===============================================================
10145  /// Activate the generic bifurcation ///tracking system by changing the
10146  /// assembly handler and initialising it using the parameter addressed by
10147  /// parameter_pt.
10148  //============================================================
10149  void Problem::activate_bifurcation_tracking(double* const& parameter_pt,
10150  const DoubleVector& eigenvector,
10151  const bool& block_solve)
10152  {
10153  // Reset the assembly handler to default
10155  // Set the new assembly handler. Note that the constructor actually
10156  // solves the original problem to get some initial conditions, but
10157  // this is OK because the RHS is always evaluated before assignment.
10158  Assembly_handler_pt = new FoldHandler(this, parameter_pt, eigenvector);
10159 
10160  // If we are using a block solver, we must set the linear solver pointer
10161  // to the block fold solver. The present linear solver is
10162  // used by the block solver and so must be passed as an argument.
10163  // The destructor of the Fold handler returns the linear
10164  // solver to the original non-block version.
10165  if (block_solve)
10166  {
10168  }
10169  }
10170 
10171 
10172  //===============================================================
10173  /// Activate the generic bifurcation ///tracking system by changing the
10174  /// assembly handler and initialising it using the parameter addressed by
10175  /// parameter_pt.
10176  //============================================================
10177  void Problem::activate_bifurcation_tracking(double* const& parameter_pt,
10178  const DoubleVector& eigenvector,
10179  const DoubleVector& normalisation,
10180  const bool& block_solve)
10181  {
10182  // Reset the assembly handler to default
10184  // Set the new assembly handler. Note that the constructor actually
10185  // solves the original problem to get some initial conditions, but
10186  // this is OK because the RHS is always evaluated before assignment.
10188  new FoldHandler(this, parameter_pt, eigenvector, normalisation);
10189 
10190  // If we are using a block solver, we must set the linear solver pointer
10191  // to the block fold solver. The present linear solver is
10192  // used by the block solver and so must be passed as an argument.
10193  // The destructor of the Fold handler returns the linear
10194  // solver to the original non-block version.
10195  if (block_solve)
10196  {
10198  }
10199  }
10200 
10201 
10202  //==================================================================
10203  /// Activate the pitchfork tracking system by changing the assembly
10204  /// handler and initialising it using the parameter addressed
10205  /// by parameter_pt and a symmetry vector. The boolean flag is
10206  /// used to specify whether a block solver is used, default is true.
10207  //===================================================================
10208  void Problem::activate_pitchfork_tracking(double* const& parameter_pt,
10209  const DoubleVector& symmetry_vector,
10210  const bool& block_solve)
10211  {
10212  // Reset the assembly handler to default
10214 
10215  // Set the new assembly handler. Note that the constructor actually
10216  // solves the original problem to get some initial conditions, but
10217  // this is OK because the RHS is always evaluated before assignment.
10219  this, this->assembly_handler_pt(), parameter_pt, symmetry_vector);
10220 
10221  // If we are using a block solver, we must set the linear solver pointer
10222  // to the block pitchfork solver. The present linear solver is
10223  // used by the block solver and so must be passed as an argument.
10224  // The destructor of the PitchFork handler returns the linear
10225  // solver to the original non-block version.
10226  if (block_solve)
10227  {
10229  }
10230  }
10231 
10232 
10233  //============================================================
10234  /// Activate the hopf tracking system by changing the assembly
10235  /// handler and initialising it using the parameter addressed
10236  /// by parameter_pt.
10237  //============================================================
10238  void Problem::activate_hopf_tracking(double* const& parameter_pt,
10239  const bool& block_solve)
10240  {
10241  // Reset the assembly handler to default
10243  // Set the new assembly handler. Note that the constructor actually
10244  // solves the original problem to get some initial conditions, but
10245  // this is OK because the RHS is always evaluated before assignment.
10246  Assembly_handler_pt = new HopfHandler(this, parameter_pt);
10247 
10248  // If we are using a block solver, we must set the linear solver pointer
10249  // to the block hopf solver. The present linear solver is
10250  // used by the block solver and so must be passed as an argument.
10251  // The destructor of the Hopf handler returns the linear
10252  // solver to the original non-block version.
10253  if (block_solve)
10254  {
10256  }
10257  }
10258 
10259 
10260  //============================================================
10261  /// Activate the hopf tracking system by changing the assembly
10262  /// handler and initialising it using the parameter addressed
10263  /// by parameter_pt and the frequency and null vectors
10264  /// specified.
10265  //============================================================
10266  void Problem::activate_hopf_tracking(double* const& parameter_pt,
10267  const double& omega,
10268  const DoubleVector& null_real,
10269  const DoubleVector& null_imag,
10270  const bool& block_solve)
10271  {
10272  // Reset the assembly handler to default
10274  // Set the new assembly handler. Note that the constructor actually
10275  // solves the original problem to get some initial conditions, but
10276  // this is OK because the RHS is always evaluated before assignment.
10278  new HopfHandler(this, parameter_pt, omega, null_real, null_imag);
10279 
10280  // If we are using a block solver, we must set the linear solver pointer
10281  // to the block hopf solver. The present linear solver is
10282  // used by the block solver and so must be passed as an argument.
10283  // The destructor of the Hopf handler returns the linear
10284  // solver to the original non-block version.
10285  if (block_solve)
10286  {
10288  }
10289  }
10290 
10291 
10292  //===============================================================
10293  /// Reset the assembly handler to default
10294  //===============================================================
10296  {
10297  // If we have a non-default handler
10299  {
10300  // Delete the current assembly handler
10301  delete Assembly_handler_pt;
10302  // Reset the assembly handler
10304  }
10305  }
10306 
10307  //===================================================================
10308  /// This function takes one step of length ds in pseudo-arclength.The
10309  /// argument parameter_pt is a pointer to the parameter (global variable)
10310  /// that is being traded for arc-length. The function returns the next desired
10311  /// arc-length according to criteria based upon the desired number of Newton
10312  /// Iterations per solve.
10313  //=====================================================================
10314  double Problem::arc_length_step_solve(double* const& parameter_pt,
10315  const double& ds,
10316  const unsigned& max_adapt)
10317  {
10318  // First check that we shouldn't use the other interface
10319  // by checking that the parameter isn't already stored as data
10320  if (does_pointer_correspond_to_problem_data(parameter_pt))
10321  {
10322  std::ostringstream error_message;
10323  error_message
10324  << "The parameter addressed by " << parameter_pt << " with the value "
10325  << *parameter_pt
10326  << "\n is supposed to be used for arc-length contiunation,\n"
10327  << " but it is stored in a Data object used by the problem.\n\n"
10328  << "This is bad for two reasons:\n"
10329  << "1. If it's a variable in the problem, it must already have an\n"
10330  "associated equation, so it can't be used for continuation;\n"
10331  << "2. The problem data will be reorganised in memory during "
10332  "continuation,\n"
10333  << " which means that the pointer will become invalid.\n\n"
10334  << "If you are sure that this is what you want to do you must:\n"
10335  << "A. Ensure that the value is pinned (don't worry we'll shout again "
10336  "if not)\n"
10337  << "B. Use the alternative interface\n"
10338  << " Problem::arc_length_step_solve(Data*,unsigned,...)\n"
10339  << " which uses a pointer to the data object and not the raw double "
10340  "pointer."
10341  << std::endl;
10342  throw OomphLibError(
10343  error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
10344  }
10345 
10346 
10347  // If we are using the continuation timestepper
10349  {
10350  // Has the timestepper already been added to the problem
10351  bool continuation_time_stepper_added = false;
10352  const unsigned n_time_steppers = this->ntime_stepper();
10353  for (unsigned i = 0; i < n_time_steppers; i++)
10354  {
10356  {
10357  continuation_time_stepper_added = true;
10358  break;
10359  }
10360  }
10361 
10362  // If not add it
10363  if (!continuation_time_stepper_added)
10364  {
10365  oomph_info << "Adding the continuation time stepper\n";
10367  }
10368 
10369  // Need to treat case of eigenproblems and bifurcation detection/tracking
10370  // here
10371 
10372  // Backup the current timesteppers for each mesh!
10373 
10374 
10375  // If an arc length step has not been taken then set the timestepper
10376  if (!Arc_length_step_taken)
10377  {
10378  // Set the continuation timestepper for all data in the problem
10381  << " equation numbers allocated for continuation\n";
10382  }
10383 
10384  } // End of continuation time stepper case
10385 
10386 
10387  // Just call the helper function (parameter is not from data)
10388  return arc_length_step_solve_helper(parameter_pt, ds, max_adapt);
10389  }
10390 
10391 
10392  //===================================================================
10393  /// This function takes one step of length ds in pseudo-arclength.The
10394  /// argument data_pt is a pointer to the data that holds the
10395  /// parameter (global variable)
10396  /// that is being traded for arc-length. The exact value is located at
10397  /// the location given by data_index.
10398  /// The function returns the next desired
10399  /// arc-length according to criteria based upon the desired number of Newton
10400  /// Iterations per solve.
10401  //=====================================================================
10402  double Problem::arc_length_step_solve(Data* const& data_pt,
10403  const unsigned& data_index,
10404  const double& ds,
10405  const unsigned& max_adapt)
10406  {
10407  // Firstly check that the data is pinned
10408  if (!data_pt->is_pinned(data_index))
10409  {
10410  std::ostringstream error_stream;
10411  error_stream << "The value at index " << data_index
10412  << " in the data object to be used for continuation\n"
10413  << "is not pinned, which means that it is already a\n"
10414  << "variable in the problem "
10415  << "and cannot be used for continuation.\n\n"
10416  << "Please correct your formulation by either:\n"
10417  << "A. Pinning the value"
10418  << "\n or \n"
10419  << "B. Using a different parameter for continuation"
10420  << std::endl;
10421  throw OomphLibError(
10422  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
10423  }
10424 
10425 
10426  // If we are using the continuation timestepper
10428  {
10429  // Has the timestepper already been added to the problem
10430  bool continuation_time_stepper_added = false;
10431  const unsigned n_time_steppers = this->ntime_stepper();
10432  for (unsigned i = 0; i < n_time_steppers; i++)
10433  {
10435  {
10436  continuation_time_stepper_added = true;
10437  break;
10438  }
10439  }
10440 
10441  // If not add it
10442  if (!continuation_time_stepper_added)
10443  {
10444  oomph_info << "Adding the continuation time stepper\n";
10446  }
10447 
10448  // Need to treat case of eigenproblems and bifurcation detection/tracking
10449  // here
10450 
10451 
10452  // Backup the current timesteppers for each mesh!
10453 
10454 
10455  // If an arc length step has not been taken then set the timestepper
10456  if (!Arc_length_step_taken)
10457  {
10458  // Set the continuation timestepper for all data in the problem
10461  << " equation numbers allocated for continuation\n";
10462  }
10463 
10464 
10465  } // End of continuation time stepper case
10466 
10467 
10468  // Now make a pointer to the (newly allocated) data object
10469  double* parameter_pt = data_pt->value_pt(data_index);
10470  // Call the helper function, this will change the parameter_pt if
10471  // the data storage is changed (if the timestepper has to be changed,
10472  // which happens if this is the first time that a continuation step is
10473  // taken)
10474  // ALH: Don't think this is true because it has happened above....
10475  return arc_length_step_solve_helper(parameter_pt, ds, max_adapt);
10476  }
10477 
10478  //======================================================================
10479  /// Private helper function that is used to set the appropriate
10480  /// pinned values for continuation. If the data is pinned, the its
10481  /// current value is always the same as the original value and
10482  /// the derivative is always zero. If these are not set properly
10483  /// then interpolation and projection in spatial adaptivity will
10484  /// not give the best answers.
10485  //=====================================================================
10487  {
10488  // Set the consistent values for the global mesh
10491 
10492  // Deal with the spine meshes additional numbering separately
10493  const unsigned n_sub_mesh = this->nsub_mesh();
10494  // If there is only one mesh
10495  if (n_sub_mesh == 0)
10496  {
10497  if (SpineMesh* const spine_mesh_pt = dynamic_cast<SpineMesh*>(Mesh_pt))
10498  {
10499  spine_mesh_pt->set_consistent_pinned_spine_values_for_continuation(
10501  }
10502  // If it's a triangle mesh the we need to set the
10503  }
10504  // Otherwise loop over the sub meshes
10505  else
10506  {
10507  // Assign global equation numbers first
10508  for (unsigned i = 0; i < n_sub_mesh; i++)
10509  {
10510  if (SpineMesh* const spine_mesh_pt =
10511  dynamic_cast<SpineMesh*>(Sub_mesh_pt[i]))
10512  {
10513  spine_mesh_pt->set_consistent_pinned_spine_values_for_continuation(
10515  }
10516  }
10517  }
10518 
10519  // Also set time stepper for global data
10520  const unsigned n_global = Global_data_pt.size();
10521  for (unsigned i = 0; i < n_global; ++i)
10522  {
10524  }
10525  }
10526 
10527 
10528  //===================================================================
10529  /// This function takes one step of length ds in pseudo-arclength.The
10530  /// argument parameter_pt is a pointer to the parameter (global variable)
10531  /// that is being traded for arc-length. The function returns the next desired
10532  /// arc-length according to criteria based upon the desired number of Newton
10533  /// Iterations per solve.
10534  //=====================================================================
10535  double Problem::arc_length_step_solve_helper(double* const& parameter_pt,
10536  const double& ds,
10537  const unsigned& max_adapt)
10538  {
10539  //----------------------MAKE THE PROBLEM STEADY-----------------------
10540  // Loop over the timesteppers and make them (temporarily) steady.
10541  // We can only do continuation for steady problems!
10542  unsigned n_time_steppers = ntime_stepper();
10543  // Vector of bools to store the is_steady status of the various
10544  // timesteppers when we came in here
10545  std::vector<bool> was_steady(n_time_steppers);
10546 
10547  // Loop over them all and make them (temporarily) static
10548  for (unsigned i = 0; i < n_time_steppers; i++)
10549  {
10550  was_steady[i] = time_stepper_pt(i)->is_steady();
10552  }
10553 
10554 
10555  // Max number of solves
10556  unsigned max_solve = max_adapt + 1;
10557  // Storage for newton steps in each adaptation
10558  unsigned max_count_in_adapt_loop = 0;
10559 
10560 
10561  //----SET UP MEMORY FOR QUANTITIES THAT ARE REQUIRED OUTSIDE THE LOOP----
10562 
10563  // Assign memory for solutions of the equations Jz = du/dparameter
10564  // This is needed here (outside the loop), so that we can save on
10565  // one linear solve when calculating the derivatives wrt the arc-length
10566  DoubleVector z;
10567 
10568 
10569  // Store sign of the Jacobian, used for bifurcation detection
10570  // If this is the first time that we are calling the arc-length solver,
10571  // this should not be used.
10572  int previous_sign = Sign_of_jacobian;
10573 
10574  // Flag to indicate a sign change
10575  bool SIGN_CHANGE = false;
10576 
10577 
10578  // Adaptation loop
10579  for (unsigned isolve = 0; isolve < max_solve; ++isolve)
10580  {
10581  // Only adapt after the first solve has been done
10582  if (isolve > 0)
10583  {
10584  unsigned n_refined;
10585  unsigned n_unrefined;
10586 
10587  // Adapt problem
10588  adapt(n_refined, n_unrefined);
10589 
10590 #ifdef OOMPH_HAS_MPI
10591  // Adaptation only converges if ALL the processes have no
10592  // refinement or unrefinement to perform
10593  unsigned total_refined = 0;
10594  unsigned total_unrefined = 0;
10596  {
10597  MPI_Allreduce(&n_refined,
10598  &total_refined,
10599  1,
10600  MPI_UNSIGNED,
10601  MPI_SUM,
10602  this->communicator_pt()->mpi_comm());
10603  n_refined = total_refined;
10604  MPI_Allreduce(&n_unrefined,
10605  &total_unrefined,
10606  1,
10607  MPI_UNSIGNED,
10608  MPI_SUM,
10609  this->communicator_pt()->mpi_comm());
10610  n_unrefined = total_unrefined;
10611  }
10612 #endif
10613 
10614  oomph_info << "---> " << n_refined << " elements were refined, and "
10615  << n_unrefined << " were unrefined"
10616 #ifdef OOMPH_HAS_MPI
10617  << ", in total (over all processors).\n";
10618 #else
10619  << ".\n";
10620 #endif
10621 
10622 
10623  // Check convergence of adaptation cycle
10624  if ((n_refined == 0) && (n_unrefined == 0))
10625  {
10626  oomph_info << "\n \n Solution is fully converged in "
10627  << "Problem::newton_solver(). \n \n ";
10628  break;
10629  }
10630  }
10631 
10632  //----------SAVE THE INITIAL VALUES, IN CASE THE STEP FAILS-----------
10633 
10634  // Find the number of local dofs
10635  unsigned ndof_local = Dof_distribution_pt->nrow_local();
10636 
10637  // Only need to do this in the first loop
10638  if (isolve == 0)
10639  {
10641  {
10642  // Safety check, set up the array of dof derivatives, if necessary
10643  // The distribution is the same as the (natural) distribution of the
10644  // dofs
10645  if (Dof_derivative.size() != ndof_local)
10646  {
10647  Dof_derivative.resize(ndof_local, 0.0);
10648  }
10649 
10650  // Safety check, set up the array of curren values, if necessary
10651  // Again the distribution reflects the (natural) distribution of the
10652  // dofs
10653  if (Dof_current.size() != ndof_local)
10654  {
10655  Dof_current.resize(ndof_local);
10656  }
10657  }
10658 
10659  // Save the current value of the parameter
10660  Parameter_current = *parameter_pt;
10661 
10662  // Save the current values of the degrees of freedom
10663  for (unsigned long l = 0; l < ndof_local; l++)
10664  {
10665  dof_current(l) = *Dof_pt[l];
10666  }
10667 
10668  // Set the value of ds_current
10669  Ds_current = ds;
10670  }
10671 
10672  // Counter for the number of newton steps
10673  unsigned count = 0;
10674 
10675  // Flag to indicate a successful step
10676  bool STEP_REJECTED = false;
10677 
10678 
10679  // Set the appropriate initial conditions for the pinned data
10681  {
10683  }
10684 
10685  // Loop around the step in arc-length
10686  do
10687  {
10688  // Check that the step has not fallen below the minimum tolerance
10689  if (std::fabs(Ds_current) < Minimum_ds)
10690  {
10691  std::ostringstream error_message;
10692  error_message << "DESIRED ARC-LENGTH STEP " << Ds_current
10693  << " HAS FALLEN BELOW MINIMUM TOLERANCE, " << Minimum_ds
10694  << std::endl;
10695 
10696  throw OomphLibError(error_message.str(),
10697  OOMPH_CURRENT_FUNCTION,
10698  OOMPH_EXCEPTION_LOCATION);
10699  }
10700 
10701  // Assume that we shall accept the step
10702  STEP_REJECTED = false;
10703 
10704  // Set initial value of the parameter
10706 
10707  // Perform any actions...
10708  actions_after_parameter_increase(parameter_pt);
10709 
10710  Ds_current = (*parameter_pt - Parameter_current) / Parameter_derivative;
10711 
10712  // Loop over the (local) variables and set their initial values
10713  for (unsigned long l = 0; l < ndof_local; l++)
10714  {
10715  *Dof_pt[l] = dof_current(l) + dof_derivative(l) * Ds_current;
10716  }
10717 
10718  // Actually do the newton solve stage for the continuation problem
10719  try
10720  {
10721  count = newton_solve_continuation(parameter_pt, z);
10722  }
10723  // Catch any exceptions thrown in the Newton solver
10724  catch (NewtonSolverError& error)
10725  {
10726  // Check whether it's the linear solver
10727  if (error.linear_solver_error())
10728  {
10729  std::ostringstream error_stream;
10730  error_stream << std::endl
10731  << "USER-DEFINED ERROR IN NEWTON SOLVER " << std::endl;
10732  oomph_info << "ERROR IN THE LINEAR SOLVER" << std::endl;
10733  throw OomphLibError(error_stream.str(),
10734  OOMPH_CURRENT_FUNCTION,
10735  OOMPH_EXCEPTION_LOCATION);
10736  }
10737  // Otherwise mark the step as having failed
10738  else
10739  {
10740  oomph_info << "STEP REJECTED DUE TO NEWTON SOLVER --- TRYING AGAIN"
10741  << std::endl;
10742  STEP_REJECTED = true;
10743  // Let's take a smaller step
10744  Ds_current *= (2.0 / 3.0);
10745  }
10746  }
10747  catch (InvertedElementError const& error)
10748  {
10749  oomph_info
10750  << "STEP REJECTED DUE TO INVERTED ELEMENTS --- TRYING AGAIN"
10751  << std::endl;
10752  STEP_REJECTED = true;
10753  // Let's take a smaller step
10754  Ds_current *= (2.0 / 3.0);
10755  }
10756  } while (STEP_REJECTED); // continue until a step is accepted
10757 
10758  // Set the maximum count
10759  if (count > max_count_in_adapt_loop)
10760  {
10761  max_count_in_adapt_loop = count;
10762  }
10763  } /// end of adaptation loop
10764 
10765  // Only recalculate the derivatives if there has been a Newton solve
10766  // If not, the previous values should be close enough
10767  if (max_count_in_adapt_loop > 0)
10768  {
10769  //--------------------CHECK FOR POTENTIAL BIFURCATIONS-------------
10771  {
10772  // If the sign of the jacobian is zero issue a warning
10773  if (Sign_of_jacobian == 0)
10774  {
10775  std::string error_message =
10776  "The sign of the jacobian is zero after a linear solve\n";
10777  error_message += "Either the matrix is singular (unlikely),\n";
10778  error_message += "or the linear solver cannot compute the "
10779  "determinant of the matrix;\n";
10780  error_message += "e.g. an iterative linear solver.\n";
10781  error_message +=
10782  "If the latter, bifurcation detection must be via an eigensolver\n";
10783  OomphLibWarning(error_message,
10784  "Problem::arc_length_step_solve",
10785  OOMPH_EXCEPTION_LOCATION);
10786  }
10787  // If this is the first step, we cannot rely on the previous value
10788  // of the jacobian so set the previous sign to the present sign
10789  if (!Arc_length_step_taken)
10790  {
10791  previous_sign = Sign_of_jacobian;
10792  }
10793  // If we have detected a sign change in the last converged Jacobian,
10794  // it must be a turning point or bifurcation
10795  if (Sign_of_jacobian != previous_sign)
10796  {
10797  // There has been, at least, one sign change
10799 
10800  // The sign has changed this time
10801  SIGN_CHANGE = true;
10802 
10803  // Calculate the dot product of the approximate null vector
10804  // of the Jacobian matrix ((badly) approximated by z)
10805  // and the vectors of derivatives of the residuals wrt the
10806  // global parameter
10807  // If this is small it is a bifurcation rather than a turning point.
10808  // Get the derivative wrt global parameter
10809  // DoubleVector dparam;
10810  // get_derivative_wrt_global_parameter(parameter_pt,dparam);
10811  // Calculate the dot product
10812  // double dot=0.0;
10813  // for(unsigned long n=0;n<n_dofs;++n) {dot += dparam[n]*z[n];}
10814  // z.dot(dparam);
10815 
10816  // Write the output message
10817  std::ostringstream message;
10818  message
10819  << "-----------------------------------------------------------";
10820  message << std::endl
10821  << "SIGN CHANGE IN DETERMINANT OF JACOBIAN: " << std::endl;
10822  message << "BIFURCATION OR TURNING POINT DETECTED BETWEEN "
10823  << Parameter_current << " AND " << *parameter_pt << std::endl;
10824  // message << "APPROXIMATE DOT PRODUCT : " << dot << "," << std::endl;
10825  // message << "IF CLOSE TO ZERO WE HAVE A BIFURCATION; ";
10826  // message << "OTHERWISE A TURNING POINT" << std::endl;
10827  message
10828  << "-----------------------------------------------------------"
10829  << std::endl;
10830 
10831  // Write the message to standard output
10832  oomph_info << message.str();
10833 
10834  // Open the information file for appending
10835  std::ofstream bifurcation_info("bifurcation_info",
10836  std::ios_base::app);
10837  // Write the message to the file
10838  bifurcation_info << message.str();
10839  bifurcation_info.close();
10840  }
10841  }
10842 
10843  // Calculate the derivatives required for the next stage of continuation
10844  // In this we pass the last value of z (i.e. approximation)
10846  {
10848  }
10849  // Or use finite differences
10850  else
10851  {
10853  }
10854 
10855  // If it's the first step then the value of the next step should
10856  // be the change in parameter divided by the parameter derivative
10857  // to obtain approximately the same parameter change
10858  if (!Arc_length_step_taken)
10859  {
10860  Ds_current = (*parameter_pt - Parameter_current) / Parameter_derivative;
10861  }
10862 
10863  // We have taken our first step
10864  Arc_length_step_taken = true;
10865  }
10866  // If there has not been a newton step then we still need to estimate
10867  // the derivatives in the arc length direction
10868  else
10869  {
10870  // Default is to calculate the continuation derivatives by solving the
10871  // linear system. We must do this to ensure that the derivatives are in
10872  // sync It could lead to problems near turning points when we should
10873  // really be solving an eigenproblem, but seems OK so far!
10874 
10875  // Save the current sign of the jacobian
10876  int temp_sign = Sign_of_jacobian;
10877 
10878  // Calculate the continuation derivatives, which includes a solve
10879  // of the linear system if not using finite differences
10881  {
10882  calculate_continuation_derivatives(parameter_pt);
10883  }
10884  // Otherwise use finite differences
10885  else
10886  {
10888  }
10889 
10890  // Reset the sign of the jacobian, just in case the sign has changed when
10891  // solving the continuation derivatives. The sign change will be picked
10892  // up on the next continuation step.
10893  Sign_of_jacobian = temp_sign;
10894  }
10895 
10896  // Reset the is_steady status of all timesteppers that
10897  // weren't already steady when we came in here and reset their
10898  // weights
10899  for (unsigned i = 0; i < n_time_steppers; i++)
10900  {
10901  if (!was_steady[i])
10902  {
10904  }
10905  }
10906 
10907  // If we are trying to find a bifurcation and the first sign change
10908  // has occured, use bisection
10911  {
10912  // If there has been a sign change we need to half the step size
10913  // and reverse the direction
10914  if (SIGN_CHANGE)
10915  {
10916  Ds_current *= -0.5;
10917  }
10918  // Otherwise
10919  else
10920  {
10921  // The size of the bracketed interval is always
10922  // 2ds - Ds_current (this will work even if the original step failed)
10923  // We want our new step size to be half this
10924  Ds_current = ds - 0.5 * Ds_current;
10925  }
10926  // Return the desired value of the step
10927  return Ds_current;
10928  }
10929 
10930  // If fewer than the desired number of Newton Iterations, increase the step
10931  if (max_count_in_adapt_loop < Desired_newton_iterations_ds)
10932  {
10933  return Ds_current * 1.5;
10934  }
10935  // If more than the desired number of Newton Iterations, reduce the step
10936  if (max_count_in_adapt_loop > Desired_newton_iterations_ds)
10937  {
10938  return Ds_current * (2.0 / 3.0);
10939  }
10940  // Otherwise return the step just taken
10941  return Ds_current;
10942  }
10943 
10944 
10945  //=======================================================================
10946  /// Take an explicit timestep of size dt
10947  //======================================================================
10948  void Problem::explicit_timestep(const double& dt, const bool& shift_values)
10949  {
10950 #ifdef PARANOID
10951  if (this->explicit_time_stepper_pt() == 0)
10952  {
10953  throw OomphLibError("Explicit time stepper pointer is null in problem.",
10954  OOMPH_EXCEPTION_LOCATION,
10955  OOMPH_CURRENT_FUNCTION);
10956  }
10957 #endif
10958 
10959  // Firstly we shift the time values
10960  if (shift_values)
10961  {
10963  }
10964  // Set the current value of dt, if we can
10965  if (time_pt()->ndt() > 0)
10966  {
10967  time_pt()->dt() = dt;
10968  }
10969 
10970  // Take the explicit step
10971  this->explicit_time_stepper_pt()->timestep(this, dt);
10972  }
10973 
10974 
10975  //========================================================================
10976  /// Do one timestep of size dt using Newton's method with the specified
10977  /// tolerance and linear solver defined as member data of the Problem class.
10978  /// This will be the most commonly used version
10979  /// of unsteady_newton_solve, in which the time values are always shifted
10980  /// This does not include any kind of adaptativity. If the solution fails to
10981  /// converge the program will end.
10982  //========================================================================
10983  void Problem::unsteady_newton_solve(const double& dt)
10984  {
10985  // We shift the values, so shift_values is true
10986  unsteady_newton_solve(dt, true);
10987  }
10988 
10989  //========================================================================
10990  /// Do one timestep forward of size dt using Newton's method with the
10991  /// specified tolerance and linear solver defined via member data of the
10992  /// Problem class.
10993  /// The boolean flag shift_values is used to control whether the time values
10994  /// should be shifted or not.
10995  //========================================================================
10996  void Problem::unsteady_newton_solve(const double& dt,
10997  const bool& shift_values)
10998  {
10999  // Shift the time values and the dts, according to the control flag
11000  if (shift_values)
11001  {
11003  }
11004 
11005  // Advance global time and set current value of dt
11006  time_pt()->time() += dt;
11007  time_pt()->dt() = dt;
11008 
11009  // Find out how many timesteppers there are
11010  unsigned n_time_steppers = ntime_stepper();
11011 
11012  // Loop over them all and set the weights
11013  for (unsigned i = 0; i < n_time_steppers; i++)
11014  {
11016  }
11017 
11018  // Run the individual timesteppers actions before timestep. These need to
11019  // be before the problem's actions_before_implicit_timestep so that the
11020  // boundary conditions are set consistently.
11021  for (unsigned i = 0; i < n_time_steppers; i++)
11022  {
11024  }
11025 
11026  // Now update anything that needs updating before the timestep
11027  // This could be time-dependent boundary conditions, for example.
11029 
11030  try
11031  {
11032  // Solve the non-linear problem for this timestep with Newton's method
11033  newton_solve();
11034  }
11035  // Catch any exceptions thrown in the Newton solver
11036  catch (NewtonSolverError& error)
11037  {
11038  oomph_info << std::endl
11039  << "USER-DEFINED ERROR IN NEWTON SOLVER " << std::endl;
11040  // Check whether it's the linear solver
11041  if (error.linear_solver_error())
11042  {
11043  oomph_info << "ERROR IN THE LINEAR SOLVER" << std::endl;
11044  }
11045  // Check to see whether we have reached Max_iterations
11046  else if (error.iterations() == Max_newton_iterations)
11047  {
11048  oomph_info << "MAXIMUM NUMBER OF ITERATIONS (" << error.iterations()
11049  << ") REACHED WITHOUT CONVERGENCE " << std::endl;
11050  }
11051  // If not, it must be that we have exceeded the maximum residuals
11052  else
11053  {
11054  oomph_info << "MAXIMUM RESIDUALS: " << error.maxres()
11055  << " EXCEEDS PREDEFINED MAXIMUM " << Max_residuals
11056  << std::endl;
11057  }
11058  // Die horribly!!
11059  std::ostringstream error_stream;
11060  error_stream << "Error occured in unsteady Newton solver. " << std::endl;
11061  throw OomphLibError(
11062  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
11063  }
11064 
11065  // Run the individual timesteppers actions, these need to be before the
11066  // problem's actions_after_implicit_timestep so that the time step is
11067  // finished before the problem does any auxiliary calculations (e.g. in
11068  // semi-implicit micromagnetics the calculation of magnetostatic field).
11069  for (unsigned i = 0; i < n_time_steppers; i++)
11070  {
11072  }
11073 
11074 
11075  // Now update anything that needs updating after the timestep
11078  }
11079 
11080  //=======================================================================
11081  /// Attempt to take one timestep forward using dt_desired. The error control
11082  /// parameter, epsilon, is used to specify the desired approximate value of
11083  /// the global error norm per timestep. The routine returns the value an
11084  /// estimate of the next value of dt that should be taken.
11085  //=======================================================================
11086  double Problem::adaptive_unsteady_newton_solve(const double& dt_desired,
11087  const double& epsilon)
11088  {
11089  // We always want to shift the time values
11090  return adaptive_unsteady_newton_solve(dt_desired, epsilon, true);
11091  }
11092 
11093 
11094  //=======================================================================
11095  /// Attempt to take one timestep forward using the dt_desired.
11096  /// This is the driver for a number of adaptive solvers. If the solution
11097  /// fails to converge at a given timestep, the routine will automatically
11098  /// halve the time step and try again, until the time step falls below the
11099  /// specified minimum value. The routine returns the value an estimate
11100  /// of the next value of dt that should be taken.
11101  /// Timestep is also rejected if the error estimate post-solve
11102  /// (computed by global_temporal_error_norm()) exceeds epsilon.
11103  /// This behaviour can be over-ruled by setting the protected
11104  /// boolean Problem::Keep_temporal_error_below_tolerance to false.
11105  //========================================================================
11106  double Problem::adaptive_unsteady_newton_solve(const double& dt_desired,
11107  const double& epsilon,
11108  const bool& shift_values)
11109  {
11110  // First, we need to backup the existing dofs, in case the timestep is
11111  // rejected
11112 
11113  // Find total number of dofs on current processor
11114  unsigned n_dof_local = dof_distribution_pt()->nrow_local();
11115 
11116  // Now set up a Vector to hold current values
11117  Vector<double> dofs_current(n_dof_local);
11118 
11119  // Load values into dofs_current
11120  for (unsigned i = 0; i < n_dof_local; i++) dofs_current[i] = dof(i);
11121 
11122  // Store the time
11123  double time_current = time_pt()->time();
11124 
11125  // Flag to detect whether the timestep has been rejected or not
11126  bool reject_timestep = 0;
11127 
11128  // Flag to detect whether any of the timesteppers are adaptive
11129  unsigned adaptive_flag = 0;
11130 
11131  // The value of the actual timestep, by default the same as desired timestep
11132  double dt_actual = dt_desired;
11133 
11134  // Find out whether any of the timesteppers are adaptive
11135  unsigned n_time_steppers = ntime_stepper();
11136  for (unsigned i = 0; i < n_time_steppers; i++)
11137  {
11138  if (time_stepper_pt(i)->adaptive_flag())
11139  {
11140  adaptive_flag = 1;
11141  break;
11142  }
11143  }
11144 
11145  // Shift the time_values according to the control flag
11146  if (shift_values)
11147  {
11149  }
11150 
11151  // This loop surrounds the adaptive time-stepping and will not be broken
11152  // until a timestep is accepted
11153  do
11154  {
11155  // Initially we assume that this step will succeed and that this dt
11156  // value is ok.
11157  reject_timestep = 0;
11158  double dt_rescaling_factor = 1.0;
11159 
11160  // Set the new time and value of dt
11161  time_pt()->time() += dt_actual;
11162  time_pt()->dt() = dt_actual;
11163 
11164  // Loop over all timesteppers and set the weights and predictor weights
11165  for (unsigned i = 0; i < n_time_steppers; i++)
11166  {
11167  // If the time_stepper is non-adaptive, this will be zero
11170  }
11171 
11172  // Now calculate the predicted values for the all data and all positions
11174 
11175  // Run the individual timesteppers actions before timestep. These need to
11176  // be before the problem's actions_before_implicit_timestep so that the
11177  // boundary conditions are set consistently.
11178  for (unsigned i = 0; i < n_time_steppers; i++)
11179  {
11181  }
11182 
11183  // Do any updates/boundary conditions changes here
11185 
11186  // Attempt to solve the non-linear system
11187  try
11188  {
11189  // Solve the non-linear problem at this timestep
11190  newton_solve();
11191  }
11192  // Catch any exceptions thrown
11193  catch (NewtonSolverError& error)
11194  {
11195  // If it's a solver error then die
11196  if (error.linear_solver_error() ||
11198  {
11199  std::string error_message = "USER-DEFINED ERROR IN NEWTON SOLVER\n";
11200  error_message += "ERROR IN THE LINEAR SOLVER\n";
11201 
11202  // Die
11203  throw OomphLibError(
11204  error_message, OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
11205  }
11206  else
11207  {
11208  // Reject the timestep, if we have an exception
11209  oomph_info << "TIMESTEP REJECTED DUE TO THE NEWTON SOLVER"
11210  << std::endl;
11211  reject_timestep = true;
11212 
11213  // Half the time step
11214  dt_rescaling_factor = Timestep_reduction_factor_after_nonconvergence;
11215  }
11216  }
11217  catch (InvertedElementError const& error)
11218  {
11219  /// Reject the timestep, if we have an exception
11220  oomph_info << "TIMESTEP REJECTED DUE TO INVERTED ELEMENTS" << std::endl;
11221  reject_timestep = true;
11222 
11223  /// Half the time step
11224  dt_rescaling_factor = Timestep_reduction_factor_after_nonconvergence;
11225  }
11226 
11227  // Run the individual timesteppers actions, these need to be before the
11228  // problem's actions_after_implicit_timestep so that the time step is
11229  // finished before the problem does any auxiliary calculations (e.g. in
11230  // semi-implicit micromagnetics the calculation of magnetostatic field).
11231  for (unsigned i = 0; i < n_time_steppers; i++)
11232  {
11234  }
11235 
11236  // Update anything that needs updating after the timestep
11238 
11239  // If we have an adapative timestepper (and we haven't already failed)
11240  // then calculate the error estimate and rescaling factor.
11241  if (adaptive_flag && !reject_timestep)
11242  {
11243  // Once timestep has been accepted can do fancy error processing
11244  // Set the error weights
11245  for (unsigned i = 0; i < n_time_steppers; i++)
11246  {
11248  }
11249 
11250  // Get a global error norm to use in adaptivity (as specified by the
11251  // problem sub-class writer). Prevent a divide by zero if the solution
11252  // gives very close to zero error. Error norm should never be negative
11253  // but use absolute value just in case.
11254  double error = std::max(std::abs(global_temporal_error_norm()), 1e-12);
11255 
11256  // Target error that we wish our next timestep to approximately produce
11257  // as a factor of the maximum error tolerance
11258  double target_error = Target_error_safety_factor * epsilon;
11259 
11260  // Calculate the scaling factor
11261  dt_rescaling_factor = std::pow(
11262  (target_error / error), (1.0 / (1.0 + time_stepper_pt()->order())));
11263 
11264  oomph_info
11265  << "- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -\n"
11266  << "Estimated timestepping error is " << error << "\n"
11267  << "Timestep scaling factor is " << dt_rescaling_factor << "\n";
11268 
11269 
11270  // Do we have to do it again?
11271  if (error > epsilon)
11272  {
11273  oomph_info << "Estimated timestepping error " << error
11274  << " exceeds tolerance " << epsilon << "\n";
11276  {
11277  oomph_info << " --> rejecting timestep.\n";
11278  reject_timestep = true;
11279  }
11280  else
11281  {
11282  oomph_info << " ...but we're not rejecting the timestep\n";
11283  }
11284  oomph_info
11285  << "Note: This behaviour can be adjusted by changing the\n"
11286  << "protected boolean\n"
11287  << " Problem::Keep_temporal_error_below_tolerance\n\n"
11288  << "Also, if you are noticing that many of your timesteps result\n"
11289  << "in error > tolerance, try reducing the target error with\n"
11290  << "respect to the error tolerance by reducing the value of\n"
11291  << "Target_error_safety_factor from its default value of 1.0\n"
11292  << "using the access function\n"
11293  << " target_error_safety_factor() = 0.5 (e.g.)\n"
11294  << "The default strategy (Target_error_safety_factor=1.0) tries\n"
11295  << "to suggest a timestep which will produce an error equal to\n"
11296  << "the error tolerance `epsilon` which risks error > tolerance\n"
11297  << "quite often. Setting the safety factor to too small a value\n"
11298  << "will make the timesteps unnecessarily small; too large will\n"
11299  << "not address the issue -- neither is optimal and a problem\n"
11300  << "dependent compromise is needed.\n"
11301  << "for more info see:\n"
11302  << " Mayr et al. (2018), p5,9, DOI:10.1016/j.finel.2017.12.002\n"
11303  << " Harrier et al. (1993), p168, ISBN:978-3-540-56670-0\n"
11304  << " Söderlind (2002), (2.7) on p5, DOI:10.1023/A:1021160023092\n";
11305  }
11306  oomph_info
11307  << "- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -\n"
11308  << std::endl;
11309 
11310 
11311  } // End of if adaptive flag
11312 
11313 
11314  // Calculate the next time step size and check it's ok
11315  // ============================================================
11316 
11317  // Calculate the possible next time step, if no error conditions
11318  // trigger.
11319  double new_dt_candidate = dt_rescaling_factor * dt_actual;
11320 
11321  // Check that the scaling factor is within the allowed range
11322  if (dt_rescaling_factor > DTSF_max_increase)
11323  {
11324  oomph_info << "Tried to increase dt by the ratio "
11325  << dt_rescaling_factor << " which is above the maximum ("
11327  << "). Attempting to increase by the maximum ratio instead."
11328  << std::endl;
11329  new_dt_candidate = DTSF_max_increase * dt_actual;
11330  }
11331  // If we have already rejected the timestep then don't do this check
11332  // because DTSF will definitely be too small.
11333  else if ((!reject_timestep) && (dt_rescaling_factor <= DTSF_min_decrease))
11334  {
11335  // Handle this special case where we want to continue anyway (usually
11336  // Minimum_dt_but_still_proceed = -1 so this has no effect).
11337  if (new_dt_candidate < Minimum_dt_but_still_proceed)
11338  {
11339  oomph_info
11340  << "Warning: Adaptation of timestep to ensure satisfaction\n"
11341  << " of error bounds during adaptive timestepping\n"
11342  << " would lower dt below \n"
11343  << " Problem::Minimum_dt_but_still_proceed="
11344  << Minimum_dt_but_still_proceed << "\n"
11345  << " ---> We're continuing with present timestep.\n"
11346  << std::endl;
11347  dt_rescaling_factor = 1.0;
11348  // ??ds shouldn't we set new_dt_candidate =
11349  // Minimum_dt_but_still_proceed here, rather than not changing dt at
11350  // all?
11351  }
11352  else
11353  {
11354  // Otherwise reject
11355  oomph_info << "Timestep would decrease by " << dt_rescaling_factor
11356  << " which is less than the minimum scaling factor "
11357  << DTSF_min_decrease << std::endl;
11358  oomph_info << "TIMESTEP REJECTED" << std::endl;
11359  reject_timestep = 1;
11360  }
11361  }
11362 
11363  // Now check that the new dt is within the allowed range
11364  if (new_dt_candidate > Maximum_dt)
11365  {
11366  oomph_info << "Tried to increase dt to " << new_dt_candidate
11367  << " which is above the maximum (" << Maximum_dt
11368  << "). I increased it to the maximum value instead.";
11369  dt_actual = Maximum_dt;
11370  }
11371  else if (new_dt_candidate < Minimum_dt)
11372  {
11373  std::ostringstream err;
11374  err << "Tried to reduce dt to " << new_dt_candidate
11375  << " which is less than the minimum dt (" << Minimum_dt << ")."
11376  << std::endl;
11377  throw OomphLibError(
11378  err.str(), OOMPH_EXCEPTION_LOCATION, OOMPH_CURRENT_FUNCTION);
11379  }
11380  else
11381  {
11382  dt_actual = new_dt_candidate;
11383  }
11384 
11385 
11387 
11388 
11389  // If we are rejecting this attempt then revert the dofs etc.
11390  if (reject_timestep)
11391  {
11392  // Reset the time
11393  time_pt()->time() = time_current;
11394 
11395  // Reload the dofs
11396  unsigned ni = dofs_current.size();
11397  for (unsigned i = 0; i < ni; i++)
11398  {
11399  dof(i) = dofs_current[i];
11400  }
11401 
11402 #ifdef OOMPH_HAS_MPI
11403  // Synchronise the solution on different processors (on each submesh)
11404  this->synchronise_all_dofs();
11405 #endif
11406 
11407  // Call all "after" actions, e.g. to handle mesh updates
11413  }
11414 
11415  }
11416  // Keep this loop going until we accept the timestep
11417  while (reject_timestep);
11418 
11419  // Once the timestep has been accepted, return the time step that should be
11420  // used next time.
11421  return dt_actual;
11422  }
11423 
11424 
11425  //=======================================================================
11426  /// Private helper function to perform
11427  /// unsteady "doubly" adaptive Newton solve: Does temporal
11428  /// adaptation first, i.e. we try to do a timestep with an increment
11429  /// of dt, and adjusting dt until the solution on the given mesh satisfies
11430  /// the temporal error measure with tolerance epsilon. Following
11431  /// this, we do up to max_adapt spatial adaptions (without
11432  /// re-examining the temporal error). If first==true, the initial conditions
11433  /// are re-assigned after the mesh adaptations.
11434  /// Shifting of time can be suppressed by overwriting the
11435  /// default value of shift (true). [Shifting must be done
11436  /// if first_timestep==true because we're constantly re-assigning
11437  /// the initial conditions; if first_timestep==true and shift==false
11438  /// shifting is performed anyway and a warning is issued.
11439  /// Pseudo-Boolean flag suppress_resolve_after_spatial_adapt [0: false;
11440  /// 1: true] does what it says.]
11441  //========================================================================
11443  const double& dt_desired,
11444  const double& epsilon,
11445  const unsigned& max_adapt,
11446  const unsigned& suppress_resolve_after_spatial_adapt_flag,
11447  const bool& first,
11448  const bool& shift_values)
11449  {
11450  // Store the initial time
11451  double initial_time = time_pt()->time();
11452 
11453  // Take adaptive timestep, adjusting dt until tolerance is satisfied
11454  double new_dt =
11455  adaptive_unsteady_newton_solve(dt_desired, epsilon, shift_values);
11456  double dt_taken = time_pt()->dt();
11457  oomph_info << "Accepted solution taken with timestep: " << dt_taken
11458  << std::endl;
11459 
11460 
11461  // Bail out straightaway if no spatial adaptation allowed
11462  if (max_adapt == 0)
11463  {
11464  oomph_info << "No spatial refinement allowed; max_adapt=0\n";
11465  return new_dt;
11466  }
11467 
11468  // Adapt problem/mesh
11469  unsigned n_refined = 0;
11470  unsigned n_unrefined = 0;
11471  adapt(n_refined, n_unrefined);
11472 
11473  // Check if mesh has been adapted on other processors
11474  Vector<int> total_ref_count(2);
11475  total_ref_count[0] = n_refined;
11476  total_ref_count[1] = n_unrefined;
11477 
11478 
11479 #ifdef OOMPH_HAS_MPI
11481  {
11482  // Sum n_refine across all processors
11483  Vector<int> ref_count(2);
11484  ref_count[0] = n_refined;
11485  ref_count[1] = n_unrefined;
11486  MPI_Allreduce(&ref_count[0],
11487  &total_ref_count[0],
11488  2,
11489  MPI_INT,
11490  MPI_SUM,
11491  communicator_pt()->mpi_comm());
11492  }
11493 #endif
11494 
11495 
11496  // Re-solve the problem if the adaptation has changed anything
11497  if ((total_ref_count[0] != 0) || (total_ref_count[1] != 0))
11498  {
11499  if (suppress_resolve_after_spatial_adapt_flag == 1)
11500  {
11501  oomph_info << "Mesh was adapted but re-solve has been suppressed."
11502  << std::endl;
11503  }
11504  else
11505  {
11506  oomph_info
11507  << "Mesh was adapted --> we'll re-solve for current timestep."
11508  << std::endl;
11509 
11510  // Reset time to what it was when we entered here
11511  // because it will be incremented again by dt_taken.
11512  time_pt()->time() = initial_time;
11513 
11514  // Shift the timesteps? No! They've been shifted already when we
11515  // called the solve with pure temporal adaptivity...
11516  bool shift = false;
11517 
11518  // Reset the inital condition on refined meshes
11519  if (first)
11520  {
11521  // Reset default set_initial_condition has been called flag to false
11523 
11524  // Reset the initial conditions
11525  oomph_info << "Re-assigning initial condition at time="
11526  << time_pt()->time() << std::endl;
11528 
11529  // This is the first timestep so shifting
11530  // has to be done following the assignment of initial conditions,
11531  // providing the default set_initial_condition function has not
11532  // been called.
11533  // In fact, unsteady_newton_solve(...) does that automatically.
11534  // We're changing the flag here to avoid warning messages.
11536  {
11537  shift = true;
11538  }
11539  }
11540 
11541  // Now take the step again on the refined mesh, using the same
11542  // timestep as used before.
11543  unsteady_newton_solve(dt_taken, max_adapt, first, shift);
11544  }
11545  }
11546  else
11547  {
11548  oomph_info << "Mesh wasn't adapted --> we'll accept spatial refinement."
11549  << std::endl;
11550  }
11551 
11552  return new_dt;
11553  }
11554 
11555 
11556  //========================================================================
11557  /// Initialise the previous values of the variables for time stepping
11558  /// corresponding to an impulsive start. Previous history for all data
11559  /// is generated by the appropriate timesteppers. Previous nodal
11560  /// positions are simply copied backwards.
11561  //========================================================================
11563  {
11564  // Assign the impulsive values in the "master" mesh
11566 
11567  // Loop over global data
11568  unsigned Nglobal = Global_data_pt.size();
11569  for (unsigned iglobal = 0; iglobal < Nglobal; iglobal++)
11570  {
11571  Global_data_pt[iglobal]
11572  ->time_stepper_pt()
11573  ->assign_initial_values_impulsive(Global_data_pt[iglobal]);
11574  }
11575  }
11576 
11577 
11578  //=======================================================================
11579  /// Assign the values for an impulsive start and also set the initial
11580  /// values of the previous dts to both be dt
11581  //======================================================================
11583  {
11584  // First initialise the dts and set the weights
11585  initialise_dt(dt);
11586  // Now call assign_initial_values_impulsive
11588  }
11589 
11590  //=======================================================================
11591  /// Return the current value of continuous time. If not Time object
11592  /// has been assigned, then throw an error
11593  //======================================================================
11594  double& Problem::time()
11595  {
11596  if (Time_pt == 0)
11597  {
11598  throw OomphLibError("Time object has not been set",
11599  OOMPH_CURRENT_FUNCTION,
11600  OOMPH_EXCEPTION_LOCATION);
11601  }
11602  else
11603  {
11604  return Time_pt->time();
11605  }
11606  }
11607 
11608  //=======================================================================
11609  /// Return the current value of continuous time. If not Time object
11610  /// has been assigned, then throw an error. Const version.
11611  //======================================================================
11612  double Problem::time() const
11613  {
11614  if (Time_pt == 0)
11615  {
11616  throw OomphLibError("Time object has not been set",
11617  OOMPH_CURRENT_FUNCTION,
11618  OOMPH_EXCEPTION_LOCATION);
11619  }
11620  else
11621  {
11622  return Time_pt->time();
11623  }
11624  }
11625 
11626 
11627  //=======================================================================
11628  /// Set all problem data to have the same timestepper (timestepper_pt).
11629  /// This is mainly used in continuation and bifurcation detection problems
11630  /// in which case the total number of unknowns may change and the changes
11631  /// to the underlying memory layout means that the Dof_pt must be
11632  /// reallocated. Thus, the function calls assign_eqn_numbers() and returns
11633  /// the number of new equation numbers.
11634  //=========================================================================
11636  TimeStepper* const& time_stepper_pt, const bool& preserve_existing_data)
11637  {
11638  // Set the timestepper for the master mesh's nodal and elemental data
11639  // to be the
11640  // continuation time stepper. This will wipe all storage other than
11641  // the 0th (present time) value at all the data objects
11643  preserve_existing_data);
11644 
11645  // Deal with the any additional mesh level timestepper data separately
11646  const unsigned n_sub_mesh = this->nsub_mesh();
11647  // If there is only one mesh
11648  if (n_sub_mesh == 0)
11649  {
11651  preserve_existing_data);
11652  }
11653  // Otherwise loop over the sub meshes
11654  else
11655  {
11656  // Assign global equation numbers first
11657  for (unsigned i = 0; i < n_sub_mesh; i++)
11658  {
11659  this->Sub_mesh_pt[i]->set_mesh_level_time_stepper(
11660  time_stepper_pt, preserve_existing_data);
11661  }
11662  }
11663 
11664  // Also set time stepper for global data
11665  const unsigned n_global = Global_data_pt.size();
11666  for (unsigned i = 0; i < n_global; ++i)
11667  {
11668  Global_data_pt[i]->set_time_stepper(time_stepper_pt,
11669  preserve_existing_data);
11670  }
11671 
11672  // We now need to reassign equations numbers because the Dof pointer
11673  // will be inappropriate because memory has been reallocated
11674 
11675 #ifdef OOMPH_HAS_MPI
11677  {
11678  std::ostringstream warning_stream;
11679  warning_stream << "This has not been comprehensively tested for "
11680  "distributed problems.\n"
11681  << "I'm sure that I need to worry about external halo and "
11682  "external elements."
11683  << std::endl;
11685  warning_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
11686  }
11687 
11688 #endif
11689 
11690  return (this->assign_eqn_numbers());
11691  }
11692 
11693 
11694  //========================================================================
11695  /// Shift all time-dependent data along for next timestep.
11696  //========================================================================
11698  {
11699  // Move the values of dt in the Time object
11700  Time_pt->shift_dt();
11701 
11702  // Only shift time values in the "master" mesh, otherwise things will
11703  // get shifted twice in complex problems
11705 
11706  // Shift global data with their own timesteppers
11707  unsigned Nglobal = Global_data_pt.size();
11708  for (unsigned iglobal = 0; iglobal < Nglobal; iglobal++)
11709  {
11710  Global_data_pt[iglobal]->time_stepper_pt()->shift_time_values(
11711  Global_data_pt[iglobal]);
11712  }
11713  }
11714 
11715 
11716  //========================================================================
11717  /// Calculate the predictions of all variables in problem
11718  //========================================================================
11720  {
11721 // Check that if we have multiple time steppers none of them want to
11722 // predict by calling an explicit timestepper (as opposed to doing
11723 // something like an explicit step by combining known history values, as
11724 // done in BDF).
11725 #ifdef PARANOID
11726  if (Time_stepper_pt.size() != 1)
11727  {
11728  for (unsigned j = 0; j < Time_stepper_pt.size(); j++)
11729  {
11730  if (time_stepper_pt()->predict_by_explicit_step())
11731  {
11732  std::string err = "Prediction by explicit step only works for "
11733  "problems with a simple time";
11734  err += "stepper. I think implementing anything more general will";
11735  err += "require a rewrite of explicit time steppers. - David";
11736  throw OomphLibError(
11737  err, OOMPH_EXCEPTION_LOCATION, OOMPH_CURRENT_FUNCTION);
11738  }
11739  }
11740  }
11741 #endif
11742 
11743 
11744  // Predict using an explicit timestepper (don't do it if adaptive = false
11745  // because pointers probably aren't set up).
11746  if (time_stepper_pt()->predict_by_explicit_step() &&
11747  time_stepper_pt()->adaptive_flag())
11748  {
11749  // Copy the time stepper's predictor pt into problem's explicit time
11750  // stepper pt (unless problem already has its own explicit time
11751  // stepper).
11753 #ifdef PARANOID
11754  if (ets_pt == 0)
11755  {
11756  std::string err = "Requested predictions by explicit step but explicit";
11757  err += " predictor pt is null.";
11758  throw OomphLibError(
11759  err, OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
11760  }
11761 
11762  if ((explicit_time_stepper_pt() != ets_pt) &&
11763  (explicit_time_stepper_pt() != 0))
11764  {
11765  throw OomphLibError("Problem has explicit time stepper other than "
11766  "predictor, not sure how to handle this yet ??ds",
11767  OOMPH_EXCEPTION_LOCATION,
11768  OOMPH_CURRENT_FUNCTION);
11769  }
11770 #endif
11771  explicit_time_stepper_pt() = ets_pt;
11772 
11773  // Backup dofs and time
11775 
11776 #ifdef PARANOID
11777  double backup_time = time();
11778 #endif
11779 
11780  // Move time back so that we are at the start of the timestep (as
11781  // explicit_timestep functions expect). This is needed because the
11782  // predictor calculations are done after unsteady newton solve has
11783  // started, and it has already moved time forwards.
11784  double dt = time_pt()->dt();
11785  time() -= dt;
11786 
11787  // Explicit step
11788  this->explicit_timestep(dt, false);
11789 
11790  // Copy predicted dofs and time to their storage slots.
11791  set_dofs(time_stepper_pt()->predictor_storage_index(), Dof_pt);
11793 
11794  // Check we got the times right
11795 #ifdef PARANOID
11796  if (std::abs(time() - backup_time) > 1e-12)
11797  {
11798  using namespace StringConversion;
11799  std::string err = "Predictor landed at the wrong time!";
11800  err += " Expected time " + to_string(backup_time, 14) + " but got ";
11801  err += to_string(time(), 14);
11802  throw OomphLibError(
11803  err, OOMPH_EXCEPTION_LOCATION, OOMPH_CURRENT_FUNCTION);
11804  }
11805 #endif
11806 
11807  // Restore dofs and time
11809  }
11810 
11811  // Otherwise we can do predictions in a more object oriented way using
11812  // whatever timestepper the data provides (this is the normal case).
11813  else
11814  {
11815  // Calculate all predictions in the "master" mesh
11817 
11818  // Calculate predictions for global data with their own timesteppers
11819  unsigned Nglobal = Global_data_pt.size();
11820  for (unsigned iglobal = 0; iglobal < Nglobal; iglobal++)
11821  {
11822  Global_data_pt[iglobal]->time_stepper_pt()->calculate_predicted_values(
11823  Global_data_pt[iglobal]);
11824  }
11825  }
11826 
11827  // If requested then copy the predicted value into the current time data
11828  // slots, ready for the newton solver to use as an initial guess.
11830  {
11831  // Not sure I know enough about distributed problems to implement
11832  // this. Probably you just need to loop over ndof_local or something,
11833  // but I can't really test it...
11834 #ifdef OOMPH_HAS_MPI
11835  if (distributed())
11836  {
11837  throw OomphLibError("Not yet implemented for distributed problems",
11838  OOMPH_EXCEPTION_LOCATION,
11839  OOMPH_CURRENT_FUNCTION);
11840  }
11841 #endif
11842 
11843  // With multiple time steppers this is much more complex becuase you
11844  // need to check the time stepper for each data to get the
11845  // predictor_storage_index(). Do-able if you need it though.
11846  if (Time_stepper_pt.size() != 1)
11847  {
11848  std::string err = "Not implemented for multiple time steppers";
11849  throw OomphLibError(
11850  err, OOMPH_EXCEPTION_LOCATION, OOMPH_CURRENT_FUNCTION);
11851  }
11852 
11853  // Get predicted values
11854  DoubleVector predicted_dofs;
11855  get_dofs(time_stepper_pt()->predictor_storage_index(), predicted_dofs);
11856 
11857  // Update dofs at current step
11858  for (unsigned i = 0; i < ndof(); i++)
11859  {
11860  dof(i) = predicted_dofs[i];
11861  }
11862  }
11863  }
11864 
11865  //======================================================================
11866  /// Enable recycling of the mass matrix in explicit timestepping
11867  /// schemes. Useful for timestepping on fixed meshes when you want
11868  /// to avoid the linear solve phase.
11869  //=====================================================================
11871  {
11874 
11875  // If we have a discontinuous formulation set the elements to reuse
11876  // their own mass matrices
11878  {
11879  const unsigned n_element = Problem::mesh_pt()->nelement();
11880  // Loop over the other elements
11881  for (unsigned e = 0; e < n_element; e++)
11882  {
11883  // Cache the element
11884  DGElement* const elem_pt =
11885  dynamic_cast<DGElement*>(Problem::mesh_pt()->element_pt(e));
11886  elem_pt->enable_mass_matrix_reuse();
11887  }
11888  }
11889  }
11890 
11891  //======================================================================
11892  /// Turn off the recyling of the mass matrix in explicit
11893  /// time-stepping schemes
11894  //======================================================================
11896  {
11899 
11900  // If we have a discontinuous formulation set the element-level
11901  // function
11903  {
11904  const unsigned n_element = Problem::mesh_pt()->nelement();
11905  // Loop over the other elements
11906  for (unsigned e = 0; e < n_element; e++)
11907  {
11908  // Cache the element
11909  DGElement* const elem_pt =
11910  dynamic_cast<DGElement*>(Problem::mesh_pt()->element_pt(e));
11911  elem_pt->disable_mass_matrix_reuse();
11912  }
11913  }
11914  }
11915 
11916 
11917  //=========================================================================
11918  /// Copy Data values, nodal positions etc from specified problem.
11919  /// Note: This is not a copy constructor. We assume that the current
11920  /// and the "original" problem have both been created by calling
11921  /// the same problem constructor so that all Data objects,
11922  /// time steppers etc. in the two problems are completely independent.
11923  /// This function copies the nodal, internal and global values
11924  /// and the time parameters from the original problem into "this"
11925  /// one. This functionality is required, e.g. for
11926  /// multigrid computations.
11927  //=========================================================================
11928  void Problem::copy(Problem* orig_problem_pt)
11929  {
11930  // Copy time
11931  //----------
11932 
11933  // Flag to indicate that orig problem is unsteady problem
11934  bool unsteady_flag = (orig_problem_pt->time_pt() != 0);
11935 
11936  // Copy current time and previous time increments for proper unsteady run
11937  if (unsteady_flag)
11938  {
11939  oomph_info << "Copying an unsteady problem." << std::endl;
11940  // Current time
11941  this->time_pt()->time() = orig_problem_pt->time_pt()->time();
11942  // Timesteps
11943  unsigned n_dt = orig_problem_pt->time_pt()->ndt();
11944  time_pt()->resize(n_dt);
11945  for (unsigned i = 0; i < n_dt; i++)
11946  {
11947  time_pt()->dt(i) = orig_problem_pt->time_pt()->dt(i);
11948  }
11949 
11950  // Find out how many timesteppers there are
11951  unsigned n_time_steppers = ntime_stepper();
11952 
11953  // Loop over them all and set the weights
11954  for (unsigned i = 0; i < n_time_steppers; i++)
11955  {
11957  }
11958  }
11959 
11960  // Copy nodes
11961  //-----------
11962 
11963  // Loop over submeshes:
11964  unsigned nmesh = nsub_mesh();
11965  if (nmesh == 0) nmesh = 1;
11966  for (unsigned m = 0; m < nmesh; m++)
11967  {
11968  // Find number of nodes in present mesh
11969  unsigned long n_node = mesh_pt(m)->nnode();
11970 
11971  // Check # of nodes:
11972  unsigned long n_node_orig = orig_problem_pt->mesh_pt(m)->nnode();
11973  if (n_node != n_node_orig)
11974  {
11975  std::ostringstream error_message;
11976  error_message << "Number of nodes in copy " << n_node
11977  << " not equal to the number in the original "
11978  << n_node_orig << std::endl;
11979 
11980  throw OomphLibError(error_message.str(),
11981  OOMPH_CURRENT_FUNCTION,
11982  OOMPH_EXCEPTION_LOCATION);
11983  }
11984 
11985  // Loop over the nodes
11986  for (unsigned long i = 0; i < n_node; i++)
11987  {
11988  // Try to cast to elastic node
11989  SolidNode* el_node_pt =
11990  dynamic_cast<SolidNode*>(mesh_pt(m)->node_pt(i));
11991  if (el_node_pt != 0)
11992  {
11993  SolidNode* el_node_orig_pt =
11994  dynamic_cast<SolidNode*>(orig_problem_pt->mesh_pt(m)->node_pt(i));
11995  el_node_pt->copy(el_node_orig_pt);
11996  }
11997  else
11998  {
11999  mesh_pt(m)->node_pt(i)->copy(orig_problem_pt->mesh_pt(m)->node_pt(i));
12000  }
12001  }
12002  }
12003 
12004 
12005  // Copy global data:
12006  //------------------
12007 
12008  // Number of global data
12009  unsigned n_global = Global_data_pt.size();
12010 
12011  // Check # of nodes in orig problem
12012  unsigned long n_global_orig = orig_problem_pt->nglobal_data();
12013  if (n_global != n_global_orig)
12014  {
12015  std::ostringstream error_message;
12016  error_message << "Number of global data in copy " << n_global
12017  << " not equal to the number in the original "
12018  << n_global_orig << std::endl;
12019 
12020  throw OomphLibError(
12021  error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
12022  }
12023 
12024  for (unsigned iglobal = 0; iglobal < n_global; iglobal++)
12025  {
12026  Global_data_pt[iglobal]->copy(orig_problem_pt->global_data_pt(iglobal));
12027  }
12028 
12029 
12030  // Copy internal data of elements:
12031  //--------------------------------
12032 
12033  // Loop over submeshes:
12034  for (unsigned m = 0; m < nmesh; m++)
12035  {
12036  // Loop over elements and deal with internal data
12037  unsigned n_element = mesh_pt(m)->nelement();
12038  for (unsigned e = 0; e < n_element; e++)
12039  {
12040  GeneralisedElement* el_pt = mesh_pt(m)->element_pt(e);
12041  unsigned n_internal = el_pt->ninternal_data();
12042  if (n_internal > 0)
12043  {
12044  // Check # of internals :
12045  unsigned long n_internal_orig =
12046  orig_problem_pt->mesh_pt(m)->element_pt(e)->ninternal_data();
12047  if (n_internal != n_internal_orig)
12048  {
12049  std::ostringstream error_message;
12050  error_message << "Number of internal data in copy " << n_internal
12051  << " not equal to the number in the original "
12052  << n_internal_orig << std::endl;
12053 
12054  throw OomphLibError(error_message.str(),
12055  OOMPH_CURRENT_FUNCTION,
12056  OOMPH_EXCEPTION_LOCATION);
12057  }
12058  for (unsigned i = 0; i < n_internal; i++)
12059  {
12060  el_pt->internal_data_pt(i)->copy(
12061  orig_problem_pt->mesh_pt(m)->element_pt(e)->internal_data_pt(i));
12062  }
12063  }
12064  }
12065  }
12066  }
12067 
12068  //=========================================================================
12069  /// Make and return a pointer to the copy of the problem. A virtual
12070  /// function that must be filled in by the user is they wish to perform
12071  /// adaptive refinement in bifurcation tracking or in multigrid problems.
12072  /// ALH: WILL NOT BE NECESSARY IN BIFURCATION TRACKING IN LONG RUN...
12073  //=========================================================================
12075  {
12076  std::ostringstream error_stream;
12077  error_stream
12078  << "This function must be overloaded in your specific problem, and must\n"
12079  << "create an exact copy of your problem. Usually this will be achieved\n"
12080  << "by a call to the constructor with exactly the same arguments as "
12081  "used\n";
12082 
12083  throw OomphLibError(
12084  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
12085  }
12086 
12087 
12088  //=========================================================================
12089  /// Dump refinement pattern of all refineable meshes and all generic
12090  /// Problem data to file for restart.
12091  //=========================================================================
12092  void Problem::dump(std::ofstream& dump_file) const
12093  {
12094  // Number of submeshes?
12095  unsigned n_mesh = nsub_mesh();
12096 
12097  dump_file << std::max(unsigned(1), n_mesh) << " # number of (sub)meshes "
12098  << std::endl;
12099 
12100  // Single mesh:
12101  //------------
12102  if (n_mesh == 0)
12103  {
12104  // Dump level of refinement before pruning
12105  if (TreeBasedRefineableMeshBase* mmesh_pt =
12106  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(0)))
12107  {
12108  dump_file << mmesh_pt->uniform_refinement_level_when_pruned()
12109  << " # uniform refinement when pruned " << std::endl;
12110  }
12111  else
12112  {
12113  dump_file << 0 << " # (fake) uniform refinement when pruned "
12114  << std::endl;
12115  }
12116  dump_file << 9999 << " # test flag for end of sub-meshes " << std::endl;
12117  }
12118 
12119  // Multiple submeshes
12120  //------------------
12121  else
12122  {
12123  // Loop over submeshes to dump level of refinement before pruning
12124  for (unsigned imesh = 0; imesh < n_mesh; imesh++)
12125  {
12126  if (TreeBasedRefineableMeshBase* mmesh_pt =
12127  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(imesh)))
12128  {
12129  dump_file << mmesh_pt->uniform_refinement_level_when_pruned()
12130  << " # uniform refinement when pruned " << std::endl;
12131  }
12132  else
12133  {
12134  dump_file << 0 << " # (fake) uniform refinement when pruned "
12135  << std::endl;
12136  }
12137  }
12138  dump_file << 9999 << " # test flag for end of sub-meshes " << std::endl;
12139  }
12140 
12141 #ifdef OOMPH_HAS_MPI
12142 
12143  const int my_rank = this->communicator_pt()->my_rank();
12144 
12145  // Record destination of all base elements
12146  unsigned n = Base_mesh_element_pt.size();
12147  Vector<int> local_base_element_processor(n, -1);
12148  Vector<int> base_element_processor(n, -1);
12149  for (unsigned e = 0; e < n; e++)
12150  {
12152  if (el_pt != 0)
12153  {
12154  if (!el_pt->is_halo())
12155  {
12156  local_base_element_processor[e] = my_rank;
12157  }
12158  }
12159  }
12160 
12161 
12162  // Get target for all base elements by reduction
12164  {
12165  // Check that the base elements have been associated to a processor
12166  // (the Base_mesh_elemen_pt is only used for structured meshes,
12167  // therefore, if there are no ustructured meshes as part of the
12168  // problem this container will be empty)
12169  if (n > 0)
12170  {
12171  MPI_Allreduce(&local_base_element_processor[0],
12172  &base_element_processor[0],
12173  n,
12174  MPI_INT,
12175  MPI_MAX,
12176  this->communicator_pt()->mpi_comm());
12177  }
12178  }
12179  else
12180  {
12181  // All the same...
12182  base_element_processor = local_base_element_processor;
12183  }
12184 
12185 
12186  dump_file << n << " # Number of base elements; partitioning follows.\n";
12187  for (unsigned e = 0; e < n; e++)
12188  {
12189  dump_file << base_element_processor[e] << "\n";
12190  }
12191  dump_file << "8888 #test flag for end of base element distribution\n";
12192 
12193 #endif
12194 
12195  // Single mesh:
12196  //------------
12197  if (n_mesh == 0)
12198  {
12199  // Dump single mesh refinement pattern (if mesh is refineable)
12200  if (TreeBasedRefineableMeshBase* mmesh_pt =
12201  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(0)))
12202  {
12203  mmesh_pt->dump_refinement(dump_file);
12204  }
12205 #ifdef OOMPH_HAS_TRIANGLE_LIB
12206  // Dump triangle mesh TriangulateIO which represents mesh topology
12207  TriangleMeshBase* mmesh_pt = dynamic_cast<TriangleMeshBase*>(mesh_pt(0));
12208  if (mmesh_pt != 0 && mmesh_pt->use_triangulateio_restart())
12209  {
12210 #ifdef OOMPH_HAS_MPI
12211  // Check if the mesh is distributed, if that is the case then
12212  // additional info. needs to be saved
12213  if (mmesh_pt->is_mesh_distributed())
12214  {
12215  // Dump the info. related with the distribution of the mesh
12216  mmesh_pt->dump_distributed_info_for_restart(dump_file);
12217  }
12218 #endif
12219  mmesh_pt->dump_triangulateio(dump_file);
12220  }
12221 #endif
12222  }
12223 
12224  // Multiple submeshes
12225  //------------------
12226  else
12227  {
12228  // Loop over submeshes
12229  for (unsigned imesh = 0; imesh < n_mesh; imesh++)
12230  {
12231  // Dump single mesh refinement pattern (if mesh is refineable)
12232  if (TreeBasedRefineableMeshBase* mmesh_pt =
12233  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(imesh)))
12234  {
12235  mmesh_pt->dump_refinement(dump_file);
12236  }
12237 #ifdef OOMPH_HAS_TRIANGLE_LIB
12238  // Dump triangle mesh TriangulateIO which represents mesh topology
12239  TriangleMeshBase* mmesh_pt =
12240  dynamic_cast<TriangleMeshBase*>(mesh_pt(imesh));
12241  if (mmesh_pt != 0 && mmesh_pt->use_triangulateio_restart())
12242  {
12243 #ifdef OOMPH_HAS_MPI
12244  // Check if the mesh is distributed, if that is the case then
12245  // additional info. needs to be saved
12246  if (mmesh_pt->is_mesh_distributed())
12247  {
12248  // Dump the info. related with the distribution of the mesh
12249  mmesh_pt->dump_distributed_info_for_restart(dump_file);
12250  }
12251 #endif
12252  mmesh_pt->dump_triangulateio(dump_file);
12253  }
12254 #endif
12255  } // End of loop over submeshes
12256  }
12257 
12258 
12259  // Dump time
12260  // ---------
12261 
12262  // Flag to indicate unsteady run
12263  bool unsteady_flag = (time_pt() != 0);
12264  dump_file << unsteady_flag << " # bool flag for unsteady" << std::endl;
12265 
12266  // Current time and previous time increments for proper unsteady run
12267  if (unsteady_flag)
12268  {
12269  // Current time
12270  dump_file << time_pt()->time() << " # Time " << std::endl;
12271  // Timesteps
12272  unsigned n_dt = time_pt()->ndt();
12273  dump_file << n_dt << " # Number of timesteps " << std::endl;
12274  for (unsigned i = 0; i < n_dt; i++)
12275  {
12276  dump_file << time_pt()->dt(i) << " # dt " << std::endl;
12277  }
12278  }
12279  // Dummy time and previous time increments for steady run
12280  else
12281  {
12282  // Current time
12283  dump_file << "0.0 # Dummy time from steady run " << std::endl;
12284  // Timesteps
12285  dump_file << "0 # Dummy number of timesteps from steady run" << std::endl;
12286  }
12287 
12288  // Loop over submeshes and dump their data
12289  unsigned nmesh = nsub_mesh();
12290  if (nmesh == 0) nmesh = 1;
12291  for (unsigned m = 0; m < nmesh; m++)
12292  {
12293  mesh_pt(m)->dump(dump_file);
12294  }
12295 
12296  // Dump global data
12297 
12298  // Loop over global data
12299  unsigned Nglobal = Global_data_pt.size();
12300  dump_file << Nglobal << " # number of global Data items " << std::endl;
12301  for (unsigned iglobal = 0; iglobal < Nglobal; iglobal++)
12302  {
12303  Global_data_pt[iglobal]->dump(dump_file);
12304  dump_file << std::endl;
12305  }
12306  }
12307 
12308  //=========================================================================
12309  /// Read refinement pattern of all refineable meshes and refine them
12310  /// accordingly, then read all Data and nodal position info from
12311  /// file for restart. Return flag to indicate if the restart was from
12312  /// steady or unsteady solution.
12313  //=========================================================================
12314  void Problem::read(std::ifstream& restart_file, bool& unsteady_restart)
12315  {
12316  // Check if the file is actually open as it won't be if it doesn't
12317  // exist! In that case we're almost certainly restarting the run on
12318  // a larger number of processors than the restart data was produced.
12319  // Say so and return
12320  bool restart_file_is_open = true;
12321  if (!restart_file.is_open())
12322  {
12323  std::ostringstream warn_message;
12324  warn_message << "Restart file isn't open -- I'm assuming that this is\n";
12325  warn_message << "because we're restarting on a larger number of\n";
12326  warn_message << "processor than were in use when the restart data was \n";
12327  warn_message << "dumped.\n";
12329  warn_message.str(), "Problem::read()", OOMPH_EXCEPTION_LOCATION);
12330  restart_file_is_open = false;
12331  }
12332 
12333  // Number of (sub)meshes?
12334  unsigned n_mesh = std::max(unsigned(1), nsub_mesh());
12335 
12336  std::string input_string;
12337 
12338  // Read line up to termination sign
12339  getline(restart_file, input_string, '#');
12340 
12341  // Ignore rest of line
12342  restart_file.ignore(80, '\n');
12343 
12344  // Read in number of sub-meshes
12345  unsigned n_submesh_read;
12346  n_submesh_read = std::atoi(input_string.c_str());
12347 
12348 #ifdef PARANOID
12349  if (restart_file_is_open)
12350  {
12351  if (n_submesh_read != n_mesh)
12352  {
12353  std::ostringstream error_message;
12354  error_message
12355  << "Number of sub-meshes specified in restart file, "
12356  << n_submesh_read << " doesn't \n match the my number of sub-meshes,"
12357  << n_mesh << std::endl
12358  << "Make sure all sub-meshes have been added to the global mesh\n"
12359  << "when calling the Problem::dump() function.\n";
12360  throw OomphLibError(error_message.str(),
12361  OOMPH_CURRENT_FUNCTION,
12362  OOMPH_EXCEPTION_LOCATION);
12363  }
12364  }
12365 #else
12366  // Suppress comiler warnings about non-used variable
12367  n_submesh_read++;
12368  n_submesh_read--;
12369 #endif
12370 
12371 
12372  // Read levels of refinement before pruning
12373 #ifdef OOMPH_HAS_MPI
12374  bool refine_and_prune_required = false;
12375 #endif
12376  Vector<unsigned> nrefinement_for_mesh(n_mesh);
12377  for (unsigned i = 0; i < n_mesh; i++)
12378  {
12379  // Read line up to termination sign
12380  getline(restart_file, input_string, '#');
12381 
12382  // Ignore rest of line
12383  restart_file.ignore(80, '\n');
12384 
12385  // Convert
12386  nrefinement_for_mesh[i] = std::atoi(input_string.c_str());
12387 
12388  // Get pointer to sub-mesh in incarnation as tree-based refineable mesh
12389  TreeBasedRefineableMeshBase* ref_mesh_pt =
12390  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(i));
12391 
12392  // If it's not a tree-based refineable mesh, ignore the following
12393  if (ref_mesh_pt == 0)
12394  {
12395  if (nrefinement_for_mesh[i] != 0)
12396  {
12397  std::ostringstream error_stream;
12398  error_stream << "Nonzero uniform-refinement-when-pruned specified\n"
12399  << "even though mesh is not tree-based. Odd. May want\n"
12400  << "to check this carefully before disabling this \n"
12401  << "warning/error -- most likely if/when we start to\n"
12402  << "prune unstructured meshes [though I can't see why\n"
12403  << "we would want to do this, given that they are \n"
12404  << "currently totally re-generated...]\n";
12405  throw OomphLibError(error_stream.str(),
12406  OOMPH_CURRENT_FUNCTION,
12407  OOMPH_EXCEPTION_LOCATION);
12408  }
12409  }
12410  else
12411  {
12412  // Get min and max refinement level
12413  unsigned local_min_ref = 0;
12414  unsigned local_max_ref = 0;
12415  ref_mesh_pt->get_refinement_levels(local_min_ref, local_max_ref);
12416 
12417  // Overall min refinement level over all meshes
12418  unsigned min_ref = local_min_ref;
12419 
12420 #ifdef OOMPH_HAS_MPI
12422  {
12423  // Reconcile between processors: If (e.g. following
12424  // distribution/pruning) the mesh has no elements on this
12425  // processor) then ignore its contribution to the poll of
12426  // max/min refinement levels
12427  int int_local_min_ref = local_min_ref;
12428  if (ref_mesh_pt->nelement() == 0)
12429  {
12430  int_local_min_ref = INT_MAX;
12431  }
12432  int int_min_ref = 0;
12433  MPI_Allreduce(&int_local_min_ref,
12434  &int_min_ref,
12435  1,
12436  MPI_INT,
12437  MPI_MIN,
12438  Communicator_pt->mpi_comm());
12439 
12440  // Overall min refinement level over all meshes
12441  min_ref = unsigned(int_min_ref);
12442  }
12443 #endif
12444 
12445  // Need to refine less
12446  if (nrefinement_for_mesh[i] >= min_ref)
12447  {
12448  nrefinement_for_mesh[i] -= min_ref;
12449  }
12450  }
12451 
12452 #ifdef OOMPH_HAS_MPI
12453  if (nrefinement_for_mesh[i] > 0)
12454  {
12455  refine_and_prune_required = true;
12456  }
12457 #endif
12458  }
12459 
12460 
12461  // Reconcile overall need to refine and prune (even empty
12462  // processors have to participate in some communication!)
12463 #ifdef OOMPH_HAS_MPI
12465  {
12466  unsigned local_req_flag = 0;
12467  unsigned req_flag = 0;
12468  if (refine_and_prune_required)
12469  {
12470  local_req_flag = 1;
12471  }
12472  MPI_Allreduce(&local_req_flag,
12473  &req_flag,
12474  1,
12475  MPI_UNSIGNED,
12476  MPI_MAX,
12477  Communicator_pt->mpi_comm());
12478  refine_and_prune_required = false;
12479  if (req_flag == 1)
12480  {
12481  refine_and_prune_required = true;
12482  }
12483 
12484  // If refine and prune is required make number of uniform
12485  // refinements for each mesh consistent otherwise code
12486  // hangs on "empty" processors for which no restart file exists
12487  if (refine_and_prune_required)
12488  {
12489  // This is what we have locally
12490  Vector<unsigned> local_nrefinement_for_mesh(nrefinement_for_mesh);
12491  // Synchronise over all processors with max operation
12492  MPI_Allreduce(&local_nrefinement_for_mesh[0],
12493  &nrefinement_for_mesh[0],
12494  n_mesh,
12495  MPI_UNSIGNED,
12496  MPI_MAX,
12497  Communicator_pt->mpi_comm());
12498 
12499 #ifdef PARANOID
12500  // Check it: Reconciliation should only be required for
12501  // for processors on which no restart file was opened and
12502  // for which the meshes are therefore empty
12503  bool fail = false;
12504  std::ostringstream error_message;
12505  error_message << "Number of uniform refinements was not consistent \n"
12506  << "for following meshes during restart on processor \n"
12507  << "on which restart file could be opened:\n";
12508  for (unsigned i = 0; i < n_mesh; i++)
12509  {
12510  if ((local_nrefinement_for_mesh[i] != nrefinement_for_mesh[i]) &&
12511  restart_file_is_open)
12512  {
12513  fail = true;
12514  error_message << "Sub-mesh: " << i << "; local nrefinement: "
12515  << local_nrefinement_for_mesh[i] << " "
12516  << "; global/synced nrefinement: "
12517  << nrefinement_for_mesh[i] << "\n";
12518  }
12519  }
12520  if (fail)
12521  {
12523  error_message.str(), "Problem::read()", OOMPH_EXCEPTION_LOCATION);
12524  }
12525 #endif
12526  }
12527  }
12528 #endif
12529 
12530  // Read line up to termination sign
12531  getline(restart_file, input_string, '#');
12532 
12533  // Ignore rest of line
12534  restart_file.ignore(80, '\n');
12535 
12536  // Check flag that indicates that we've read the final data
12537  unsigned tmp;
12538  tmp = std::atoi(input_string.c_str());
12539 
12540 #ifdef PARANOID
12541  if (restart_file_is_open)
12542  {
12543  if (tmp != 9999)
12544  {
12545  std::ostringstream error_message;
12546  error_message
12547  << "Error in reading restart data: Uniform refinement when pruned \n"
12548  << "flags should be followed by 9999.\n";
12549  throw OomphLibError(error_message.str(),
12550  OOMPH_CURRENT_FUNCTION,
12551  OOMPH_EXCEPTION_LOCATION);
12552  }
12553  }
12554 
12555 #else
12556  // Suppress comiler warnings about non-used variable
12557  tmp++;
12558  tmp--;
12559 #endif
12560 
12561 
12562 #ifdef OOMPH_HAS_MPI
12563 
12564  // Refine and prune if required
12565  if (refine_and_prune_required)
12566  {
12567  refine_uniformly(nrefinement_for_mesh);
12569  }
12570 
12571  // target_domain_for_local_non_halo_element[e] contains the number
12572  // of the domain [0,1,...,nproc-1] to which non-halo element e on THE
12573  // CURRENT PROCESSOR ONLY has been assigned. The order of the non-halo
12574  // elements is the same as in the Problem's mesh, with the halo
12575  // elements being skipped.
12576  Vector<unsigned> target_domain_for_local_non_halo_element;
12577 
12578  // If a restart file has been generated using code compiled without MPI
12579  // then it will not have any of the base element data.
12580  // If we try to read in that file with code that has been compied using
12581  // MPI, even if running only one processor, then it will fail here.
12582  // The ideal fix is to edit the restart file so that it contains the two
12583  // lines
12584  //
12585  // 0 # Number of base elements; partitioning follows.
12586  // 8888 # Test flag for end of base element distribution
12587  //
12588  // after the end of the sub-meshes, but before the number of elements
12589  // However, we can determine that this is the problem if n_base = 0,
12590  // so there is a little bit of logic below to catch this case
12591 
12592  // Store current location in the file (before we are about to read
12593  // in either the base mesh or number of elements of the first mesh)
12594  std::streampos position_before_base_element = restart_file.tellg();
12595  // Boolean flag used to set whether to read in base element info
12596  bool read_in_base_element_info = true;
12597 
12598  // Read line up to termination sign
12599  getline(restart_file, input_string, '#');
12600 
12601  // Ignore rest of line
12602  restart_file.ignore(80, '\n');
12603 
12604  // Get number of base elements as recorded
12605  unsigned n_base_element_read_in = atoi(input_string.c_str());
12606  unsigned nbase = Base_mesh_element_pt.size();
12607  if (restart_file_is_open)
12608  {
12609  if (n_base_element_read_in != nbase)
12610  {
12611  // If we have zero base elements the problem could be that the
12612  // restart file was generated without MPI. Issue a warning
12613  // and continue anyway
12614  if (nbase == 0)
12615  {
12616  std::ostringstream warn_message;
12617  warn_message
12618  << "The number of base elements in the mesh is 0,\n"
12619  << " but the restart file indicates that there are "
12620  << n_base_element_read_in << ".\n"
12621  << "This could be because the restart file was \n"
12622  << "generated by using code without MPI.\n"
12623  << "\n"
12624  << "The best fix is to include two additional lines\n"
12625  << "in the restart file: \n\n"
12626  << "0 # Number of base elements; partitioning follows.\n"
12627  << "8888 # Test flag for end of base element distribution\n"
12628  << "\n"
12629  << "These lines go after the flag 9999 that indicates\n"
12630  << "the end of the submesh information.\n"
12631  << "\n"
12632  << "The file will now continue to be read assuming that\n"
12633  << "the base element information is not present.\n"
12634  << "If you get strange results then please look carefully\n"
12635  << "at the restart file. The safest thing to do is to \n"
12636  << "ensure that the restart file was generated by code\n"
12637  << "compiled and run with the same parallel options.\n";
12638  OomphLibWarning(warn_message.str(),
12639  OOMPH_CURRENT_FUNCTION,
12640  OOMPH_EXCEPTION_LOCATION);
12641  // Set the skip flag to true
12642  // and rewind the file pointer
12643  read_in_base_element_info = false;
12644  restart_file.seekg(position_before_base_element);
12645  }
12646  // Otherwise throw a hard error
12647  else
12648  {
12649  std::ostringstream error_message;
12650  error_message << "About to read " << n_base_element_read_in
12651  << " base elements \n"
12652  << "though we only have " << nbase
12653  << " base elements in mesh.\n";
12654  throw OomphLibError(error_message.str(),
12655  OOMPH_CURRENT_FUNCTION,
12656  OOMPH_EXCEPTION_LOCATION);
12657  }
12658  }
12659  }
12660 
12661  // Read in the remaning base element information, if necessary
12662  if (read_in_base_element_info == true)
12663  {
12664  // Read in target_domain_for_base_element[e] for all base elements
12665  Vector<unsigned> target_domain_for_base_element(nbase);
12666  for (unsigned e = 0; e < nbase; e++)
12667  {
12668  // Read line
12669  getline(restart_file, input_string);
12670 
12671  // Get target domain
12672  target_domain_for_base_element[e] = atoi(input_string.c_str());
12673  }
12674 
12675  // Read line up to termination sign
12676  getline(restart_file, input_string, '#');
12677 
12678  // Ignore rest of line
12679  restart_file.ignore(80, '\n');
12680 
12681  // Check flag that indicates that we've read the final data
12682  tmp = std::atoi(input_string.c_str());
12683 
12684 
12685 #ifdef PARANOID
12686  if (restart_file_is_open)
12687  {
12688  if (tmp != 8888)
12689  {
12690  std::ostringstream error_message;
12691  error_message
12692  << "Error in reading restart data: Target proc for base elements \n"
12693  << "should be followed by 8888.\n";
12694  throw OomphLibError(error_message.str(),
12695  OOMPH_CURRENT_FUNCTION,
12696  OOMPH_EXCEPTION_LOCATION);
12697  }
12698  }
12699 #endif
12700 
12701  // Loop over all elements (incl. any FaceElements) and assign
12702  // target domain for all local non-halo elements and check if
12703  // load balancing is required -- no need to do this if problem is
12704  // not distributed.
12705  unsigned load_balance_required_flag = 0;
12707  {
12708  // Working with TreeBasedRefineableMeshBase mesh
12709  unsigned local_load_balance_required_flag = 0;
12710  if (dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(0)))
12711  {
12712  const int my_rank = this->communicator_pt()->my_rank();
12713  unsigned nel = mesh_pt()->nelement();
12714  for (unsigned e = 0; e < nel; e++)
12715  {
12716  GeneralisedElement* el_pt = mesh_pt()->element_pt(e);
12717  if (!el_pt->is_halo())
12718  {
12719  // Get element number (plus one) in base element enumeration
12720  unsigned el_number_in_base_mesh_plus_one =
12722 
12723  // If it's zero then we haven't found it, it may be a FaceElement
12724  // (in which case we move it to the same processor as its bulk
12725  // element
12726  if (el_number_in_base_mesh_plus_one == 0)
12727  {
12728  FaceElement* face_el_pt = dynamic_cast<FaceElement*>(el_pt);
12729  if (face_el_pt != 0)
12730  {
12731  // Get corresponding bulk element
12732  FiniteElement* bulk_el_pt = face_el_pt->bulk_element_pt();
12733 
12734  // Use its element number (plus one) in base element
12735  // enumeration
12736  el_number_in_base_mesh_plus_one =
12738 
12739  // If this is zero too we have a problem
12740  if (el_number_in_base_mesh_plus_one == 0)
12741  {
12742  throw OomphLibError(
12743  "el_number_in_base_mesh_plus_one=0 for bulk",
12744  "Problem::read()",
12745  OOMPH_EXCEPTION_LOCATION);
12746  }
12747  }
12748  }
12749 
12750  // If we've made it here then we're not dealing with a
12751  // FaceElement but with an element that doesn't exist locally
12752  // --> WTF?
12753  if (el_number_in_base_mesh_plus_one == 0)
12754  {
12755  throw OomphLibError("el_number_in_base_mesh_plus_one=0",
12756  OOMPH_CURRENT_FUNCTION,
12757  OOMPH_EXCEPTION_LOCATION);
12758  }
12759 
12760  // Assign target domain for next local non-halo element in
12761  // the order in which it's encountered in the global mesh
12762  target_domain_for_local_non_halo_element.push_back(
12763  target_domain_for_base_element[el_number_in_base_mesh_plus_one -
12764  1]);
12765 
12766  // Do elements on this processor to be moved elsewhere?
12767  if (int(target_domain_for_base_element
12768  [el_number_in_base_mesh_plus_one - 1]) != my_rank)
12769  {
12770  local_load_balance_required_flag = 1;
12771  }
12772  }
12773  }
12774 
12775  } // if (working with TreeBasedRefineableMeshBase mesh)
12776 
12777  // Get overall need to load balance by max
12778  MPI_Allreduce(&local_load_balance_required_flag,
12779  &load_balance_required_flag,
12780  1,
12781  MPI_UNSIGNED,
12782  MPI_MAX,
12783  this->communicator_pt()->mpi_comm());
12784  }
12785 
12786  // Do we need to load balance?
12787  if (load_balance_required_flag == 1)
12788  {
12789  oomph_info << "Doing load balancing after pruning\n";
12790  DocInfo doc_info;
12791  doc_info.disable_doc();
12792  bool report_stats = false;
12793  load_balance(
12794  doc_info, report_stats, target_domain_for_local_non_halo_element);
12795  oomph_info << "Done load balancing after pruning\n";
12796  }
12797  else
12798  {
12799  oomph_info << "No need for load balancing after pruning\n";
12800  }
12801  } // End of read in base element information
12802 #endif
12803 
12804 
12805  // Boolean to record if any unstructured bulk meshes have
12806  // been read in (and therefore completely re-generated, with new
12807  // elements and nodes) from disk
12808  bool have_read_unstructured_mesh = false;
12809 
12810  // Call the actions before adaptation
12812 
12813  // If there are unstructured meshes in the problem we need
12814  // to strip out any face elements that are attached to them
12815  // because restart of unstructured meshes re-creates their elements
12816  // and nodes from scratch, leading to dangling pointers from the
12817  // face elements to the old elements and nodes. This function is
12818  // virtual and (practically) empty in the Problem base class
12819  // but toggles a flag to indicate that it has been called. We can then
12820  // issue a warning below, prompting the user to consider overloading it
12821  // if the problem is found to contain unstructured bulk meshes.
12822  // Warning can be ignored if the bulk mesh is not associated with any
12823  // face elements.
12826 
12827  // Update number of submeshes
12828  n_mesh = nsub_mesh();
12829 
12830  // Single mesh:
12831  //------------
12832  if (n_mesh == 0)
12833  {
12834  // Refine single mesh (if it's refineable)
12835  if (TreeBasedRefineableMeshBase* mmesh_pt =
12836  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(0)))
12837  {
12838  // When we get in here the problem has been constructed
12839  // by the constructor and the mesh is its original unrefined
12840  // form.
12841  // RefineableMeshBase::refine(...) reads the refinement pattern from the
12842  // specified file and performs refinements until the mesh has
12843  // reached the same level of refinement as the mesh that existed
12844  // when the problem was dumped to disk.
12845  mmesh_pt->refine(restart_file);
12846  }
12847 #ifdef OOMPH_HAS_TRIANGLE_LIB
12848  // Regenerate mesh from triangulate IO if it's a triangular mesh
12849  TriangleMeshBase* mmesh_pt = dynamic_cast<TriangleMeshBase*>(mesh_pt(0));
12850  if (mmesh_pt != 0 && mmesh_pt->use_triangulateio_restart())
12851  {
12852 #ifdef OOMPH_HAS_MPI
12853  // Check if the mesh is distributed, if that is the case then
12854  // additional info. needs to be read
12855  if (mmesh_pt->is_mesh_distributed())
12856  {
12857  // Dump the info. related with the distribution of the mesh
12858  mmesh_pt->read_distributed_info_for_restart(restart_file);
12859  }
12860 #endif
12861  // The function reads the TriangulateIO data structure from the dump
12862  // file and then completely regenerates the mesh using the
12863  // data structure
12864  mmesh_pt->remesh_from_triangulateio(restart_file);
12865  have_read_unstructured_mesh = true;
12866 #ifdef OOMPH_HAS_MPI
12867  // Check if the mesh is distributed, if that is the case then we
12868  // need to re-establish the halo/haloed scheme (similar as in the
12869  // RefineableTriangleMesh::adapt() method)
12870  if (mmesh_pt->is_mesh_distributed())
12871  {
12873  this->communicator_pt(), restart_file);
12874  }
12875 #endif
12876  // Still left to update the polylines representation, that is performed
12877  // later since the nodes positions may still change when reading info.
12878  // for the mesh, see below
12879  }
12880 #endif
12881  }
12882 
12883  // Multiple submeshes
12884  //------------------
12885  else
12886  {
12887  // Loop over submeshes
12888  for (unsigned imesh = 0; imesh < n_mesh; imesh++)
12889  {
12890  // Refine single mesh (if its refineable)
12891  if (TreeBasedRefineableMeshBase* mmesh_pt =
12892  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(imesh)))
12893  {
12894  // When we get in here the problem has been constructed
12895  // by the constructor and the mesh is its original unrefined
12896  // form.
12897  // RefineableMeshBase::refine(...) reads the refinement pattern from
12898  // the specified file and performs refinements until the mesh has
12899  // reached the same level of refinement as the mesh that existed
12900  // when the problem was dumped to disk.
12901  mmesh_pt->refine(restart_file);
12902  }
12903 #ifdef OOMPH_HAS_TRIANGLE_LIB
12904  // Regenerate mesh from triangulate IO if it's a triangular mesh
12905  TriangleMeshBase* mmesh_pt =
12906  dynamic_cast<TriangleMeshBase*>(mesh_pt(imesh));
12907  if (mmesh_pt != 0 && mmesh_pt->use_triangulateio_restart())
12908  {
12909 #ifdef OOMPH_HAS_MPI
12910  // Check if the mesh is distributed, if that is the case then
12911  // additional info. needs to be read
12912  if (mmesh_pt->is_mesh_distributed())
12913  {
12914  // Dump the info. related with the distribution of the mesh
12915  mmesh_pt->read_distributed_info_for_restart(restart_file);
12916  }
12917 #endif
12918  // The function reads the TriangulateIO data structure from the dump
12919  // file and then completely regenerates the mesh using the
12920  // data structure
12921  mmesh_pt->remesh_from_triangulateio(restart_file);
12922  have_read_unstructured_mesh = true;
12923 
12924 #ifdef OOMPH_HAS_MPI
12925  // Check if the mesh is distributed, if that is the case then we
12926  // need to re-establish the halo/haloed scheme (similar as in the
12927  // RefineableTriangleMesh::adapt() method)
12928  if (mmesh_pt->is_mesh_distributed())
12929  {
12931  this->communicator_pt(), restart_file);
12932  }
12933 #endif
12934  // Still left to update the polylines representation, that is
12935  // performed later since the nodes positions may still change when
12936  // reading info. for the mesh, see below
12937  }
12938 #endif
12939  } // End of loop over submeshes
12940 
12941 
12942  // Rebuild the global mesh
12944  }
12945 
12946  // Any actions after adapt
12948 
12949  // Re-attach face elements (or whatever else needs to be done
12950  // following the total re-generation of the unstructured meshes
12953 
12954 
12955  // Issue warning:
12957  {
12958  if (have_read_unstructured_mesh)
12959  {
12962  {
12963  std::ostringstream warn_message;
12964  warn_message
12965  << "I've just read in some unstructured meshes and have, in\n"
12966  << "the process, totally re-generated their nodes and elements.\n"
12967  << "This may create dangling pointers that still point to the\n"
12968  << "old nodes and elements, e.g. because FaceElements were\n"
12969  << "attached to these meshes or pointers to nodes and elements\n"
12970  << "were stored somewhere. FaceElements should therefore be\n"
12971  << "removed before reading in these meshes, using an overloaded\n"
12972  << "version of the function\n\n"
12973  << " Problem::actions_before_read_unstructured_meshes()\n\n"
12974  << "and then re-attached using an overloaded version of\n\n"
12975  << " Problem::actions_after_read_unstructured_meshes().\n\n"
12976  << "The required content of these functions is likely to be "
12977  "similar\n"
12978  << "to the Problem::actions_before_adapt() and \n"
12979  << "Problem::actions_after_adapt() that would be required in\n"
12980  << "a spatially adaptive computation. If these functions already\n"
12981  << "exist and perform the required actions, the \n"
12982  << "actions_before/after_read_unstructured_meshes() functions\n"
12983  << "can remain empty because the former are called automatically.\n"
12984  << "In this case, this warning my be suppressed by setting the\n"
12985  << "public boolean\n\n"
12986  << " "
12987  "Problem::Suppress_warning_about_actions_before_read_"
12988  "unstructured_meshes\n\n"
12989  << "to true." << std::endl;
12990  OomphLibWarning(warn_message.str(),
12991  OOMPH_CURRENT_FUNCTION,
12992  OOMPH_EXCEPTION_LOCATION);
12993  }
12994  }
12995  }
12996 
12997  // Setup equation numbering scheme
12998  oomph_info << "\nNumber of equations in Problem::read(): "
12999  << assign_eqn_numbers() << std::endl
13000  << std::endl;
13001  // Read time info
13002  //---------------
13003  unsigned local_unsteady_restart_flag = 0;
13004  double local_time = -DBL_MAX;
13005  unsigned local_n_dt = 0;
13006 #ifdef OOMPH_HAS_MPI
13007  unsigned local_sync_needed_flag = 0;
13008 #endif
13009  Vector<double> local_dt;
13010 
13011  if (restart_file.is_open())
13012  {
13013  oomph_info << "Restart file exists" << std::endl;
13014 #ifdef OOMPH_HAS_MPI
13015  local_sync_needed_flag = 0;
13016 #endif
13017  // Read line up to termination sign
13018  getline(restart_file, input_string, '#');
13019 
13020  // Ignore rest of line
13021  restart_file.ignore(80, '\n');
13022 
13023  // Is the restart data from an unsteady run?
13024  local_unsteady_restart_flag = atoi(input_string.c_str());
13025 
13026  // Read line up to termination sign
13027  getline(restart_file, input_string, '#');
13028 
13029  // Ignore rest of line
13030  restart_file.ignore(80, '\n');
13031 
13032  // Read in initial time and set
13033  local_time = atof(input_string.c_str());
13034 
13035  // Read line up to termination sign
13036  getline(restart_file, input_string, '#');
13037 
13038  // Ignore rest of line
13039  restart_file.ignore(80, '\n');
13040 
13041  // Read & set number of timesteps
13042  local_n_dt = atoi(input_string.c_str());
13043  local_dt.resize(local_n_dt);
13044 
13045  // Read in timesteps:
13046  for (unsigned i = 0; i < local_n_dt; i++)
13047  {
13048  // Read line up to termination sign
13049  getline(restart_file, input_string, '#');
13050 
13051  // Ignore rest of line
13052  restart_file.ignore(80, '\n');
13053 
13054  // Read in initial time and set
13055  double prev_dt = atof(input_string.c_str());
13056  local_dt[i] = prev_dt;
13057  }
13058  }
13059  else
13060  {
13061  oomph_info << "Restart file does not exist" << std::endl;
13062 #ifdef OOMPH_HAS_MPI
13063  local_sync_needed_flag = 1;
13064 #endif
13065  }
13066 
13067 
13068  // No prepare global values, possibly via sync
13069  Vector<double> dt;
13070 
13071  // Do we need to sync?
13072  unsigned sync_needed_flag = 0;
13073 
13074 #ifdef OOMPH_HAS_MPI
13076  {
13077  // Get need to sync by max
13078  MPI_Allreduce(&local_sync_needed_flag,
13079  &sync_needed_flag,
13080  1,
13081  MPI_UNSIGNED,
13082  MPI_MAX,
13083  this->communicator_pt()->mpi_comm());
13084  }
13085 #endif
13086 
13087  // Synchronise
13088  if (sync_needed_flag == 1)
13089  {
13090 #ifdef OOMPH_HAS_MPI
13091 
13092 
13093 #ifdef PARANOID
13095  {
13096  std::ostringstream error_message;
13097  error_message << "Synchronisation of temporal restart data \n"
13098  << "required even though Problem hasn't been distributed "
13099  "-- very odd!\n";
13100  throw OomphLibError(error_message.str(),
13101  OOMPH_CURRENT_FUNCTION,
13102  OOMPH_EXCEPTION_LOCATION);
13103  }
13104 #endif
13105 
13106  // Get unsteady restart flag by max-based reduction
13107  unsigned unsteady_restart_flag = 0;
13108  MPI_Allreduce(&local_unsteady_restart_flag,
13109  &unsteady_restart_flag,
13110  1,
13111  MPI_UNSIGNED,
13112  MPI_MAX,
13113  this->communicator_pt()->mpi_comm());
13114 
13115  // So, is it an unsteady restart?
13116  unsteady_restart = false;
13117  if (unsteady_restart_flag == 1)
13118  {
13119  unsteady_restart = true;
13120 
13121  // Get time by max
13122  double time = -DBL_MAX;
13123  MPI_Allreduce(&local_time,
13124  &time,
13125  1,
13126  MPI_DOUBLE,
13127  MPI_MAX,
13128  this->communicator_pt()->mpi_comm());
13129  time_pt()->time() = time;
13130 
13131  // Get number of timesteps by max-based reduction
13132  unsigned n_dt = 0;
13133  MPI_Allreduce(&local_n_dt,
13134  &n_dt,
13135  1,
13136  MPI_UNSIGNED,
13137  MPI_MAX,
13138  this->communicator_pt()->mpi_comm());
13139 
13140  // Resize whatever needs resizing
13141  time_pt()->resize(n_dt);
13142  dt.resize(n_dt);
13143  if (local_dt.size() == 0)
13144  {
13145  local_dt.resize(n_dt, -DBL_MAX);
13146  }
13147 
13148  // Get timesteps increments by max-based reduction
13149  MPI_Allreduce(&local_dt[0],
13150  &dt[0],
13151  n_dt,
13152  MPI_DOUBLE,
13153  MPI_MAX,
13154  this->communicator_pt()->mpi_comm());
13155  }
13156 
13157 #else
13158 
13159  std::ostringstream error_message;
13160  error_message
13161  << "Synchronisation of temporal restart data \n"
13162  << "required even though we don't have mpi support -- very odd!\n";
13163  throw OomphLibError(
13164  error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
13165 
13166 #endif
13167  }
13168  // No sync needed -- just copy across
13169  else
13170  {
13171  unsteady_restart = false;
13172  if (local_unsteady_restart_flag == 1)
13173  {
13174  unsteady_restart = true;
13175  time_pt()->time() = local_time;
13176  time_pt()->resize(local_n_dt);
13177  dt.resize(local_n_dt);
13178  for (unsigned i = 0; i < local_n_dt; i++)
13179  {
13180  dt[i] = local_dt[i];
13181  }
13182  }
13183  }
13184 
13185  // Initialise timestep -- also sets the weights for all timesteppers
13186  // in the problem.
13187  if (unsteady_restart) initialise_dt(dt);
13188 
13189  // Loop over submeshes:
13190  unsigned nmesh = nsub_mesh();
13191  if (nmesh == 0) nmesh = 1;
13192  for (unsigned m = 0; m < nmesh; m++)
13193  {
13194  // //---------------------------------------------------------
13195  // // Keep this commented out code around to debug restarts
13196  // //---------------------------------------------------------
13197  // std::ofstream some_file;
13198  // char filename[100];
13199  // sprintf(filename,"read_mesh%i_on_proc%i.dat",m,
13200  // this->communicator_pt()->my_rank());
13201  // some_file.open(filename);
13202  // mesh_pt(m)->output(some_file);
13203  // some_file.close();
13204 
13205  // sprintf(filename,"read_mesh%i_with_haloes_on_proc%i.dat",m,
13206  // this->communicator_pt()->my_rank());
13207  // mesh_pt(m)->enable_output_of_halo_elements();
13208  // some_file.open(filename);
13209  // mesh_pt(m)->output(some_file);
13210  // mesh_pt(m)->disable_output_of_halo_elements();
13211  // some_file.close();
13212  // oomph_info << "Doced mesh " << m << " before reading\n";
13213 
13214  // sprintf(filename,"read_nodes_mesh%i_on_proc%i.dat",m,
13215  // this->communicator_pt()->my_rank());
13216  // some_file.open(filename);
13217  // unsigned nnod=mesh_pt(m)->nnode();
13218  // for (unsigned j=0;j<nnod;j++)
13219  // {
13220  // Node* nod_pt=mesh_pt(m)->node_pt(j);
13221  // unsigned n=nod_pt->ndim();
13222  // for (unsigned i=0;i<n;i++)
13223  // {
13224  // some_file << nod_pt->x(i) << " ";
13225  // }
13226  // some_file << nod_pt->is_halo() << " "
13227  // << nod_pt->nvalue() << " "
13228  // << nod_pt->hang_code() << "\n";
13229  // }
13230  // some_file.close();
13231  // oomph_info << "Doced mesh " << m << " before reading\n";
13232  // //---------------------------------------------------------
13233  // // End keep this commented out code around to debug restarts
13234  // //---------------------------------------------------------
13235 
13236  mesh_pt(m)->read(restart_file);
13237 
13238 #ifdef OOMPH_HAS_TRIANGLE_LIB
13239  // Here update the polyline representation if working with
13240  // triangle base meshes
13241  if (TriangleMeshBase* mmesh_pt =
13242  dynamic_cast<TriangleMeshBase*>(mesh_pt(m)))
13243  {
13244  // In charge of updating the polylines representation to the
13245  // current refinement/unrefinement level after restart, it
13246  // also update the shared boundaries in case of working with a
13247  // distributed mesh
13248  mmesh_pt->update_polyline_representation_from_restart();
13249  }
13250 #endif // #ifdef OOMPH_HAS_TRIANGLE_LIB
13251  }
13252 
13253  // Read global data:
13254  //------------------
13255 
13256  // Number of global data
13257  unsigned Nglobal = Global_data_pt.size();
13258 
13259  // Read line up to termination sign
13260  getline(restart_file, input_string, '#');
13261 
13262  // Ignore rest of line
13263  restart_file.ignore(80, '\n');
13264 
13265  // Check # of nodes:
13266  unsigned long check_nglobal = atoi(input_string.c_str());
13267 
13268 
13269  if (restart_file_is_open)
13270  {
13271  if (check_nglobal != Nglobal)
13272  {
13273  std::ostringstream error_message;
13274  error_message << "The number of global data " << Nglobal
13275  << " is not equal to that specified in the input file "
13276  << check_nglobal << std::endl;
13277 
13278  throw OomphLibError(error_message.str(),
13279  OOMPH_CURRENT_FUNCTION,
13280  OOMPH_EXCEPTION_LOCATION);
13281  }
13282  }
13283 
13284  for (unsigned iglobal = 0; iglobal < Nglobal; iglobal++)
13285  {
13286  Global_data_pt[iglobal]->read(restart_file);
13287  }
13288  }
13289 
13290  //===================================================================
13291  /// Set all timesteps to the same value, dt, and assign
13292  /// weights for all timesteppers in the problem.
13293  //===================================================================
13294  void Problem::initialise_dt(const double& dt)
13295  {
13296  // Initialise the timesteps in the Problem's time object
13297  Time_pt->initialise_dt(dt);
13298 
13299  // Find out how many timesteppers there are
13300  unsigned n_time_steppers = ntime_stepper();
13301 
13302  // Loop over them all and set the weights
13303  for (unsigned i = 0; i < n_time_steppers; i++)
13304  {
13306  if (time_stepper_pt(i)->adaptive_flag())
13307  {
13309  }
13310  }
13311  }
13312 
13313  //=========================================================================
13314  /// Set the value of the timesteps to be equal to the values passed in
13315  /// a vector and assign weights for all timesteppers in the problem
13316  //========================================================================
13318  {
13319  // Initialise the timesteps in the Problem's time object
13320  Time_pt->initialise_dt(dt);
13321 
13322  // Find out how many timesteppers there are
13323  unsigned n_time_steppers = ntime_stepper();
13324 
13325  // Loop over them all and set the weights
13326  for (unsigned i = 0; i < n_time_steppers; i++)
13327  {
13329  if (time_stepper_pt(i)->adaptive_flag())
13330  {
13332  }
13333  }
13334  }
13335 
13336  //========================================================
13337  /// Self-test: Check meshes and global data. Return 0 for OK
13338  //========================================================
13340  {
13341  // Initialise
13342  bool passed = true;
13343 
13344  // Are there any submeshes?
13345  unsigned Nmesh = nsub_mesh();
13346 
13347  // Just one mesh: Check it
13348  if (Nmesh == 0)
13349  {
13350  if (mesh_pt()->self_test() != 0)
13351  {
13352  passed = false;
13353  oomph_info
13354  << "\n ERROR: Failed Mesh::self_test() for single mesh in problem"
13355  << std::endl;
13356  }
13357  }
13358  // Loop over all submeshes and check them
13359  else
13360  {
13361  for (unsigned imesh = 0; imesh < Nmesh; imesh++)
13362  {
13363  if (mesh_pt(imesh)->self_test() != 0)
13364  {
13365  passed = false;
13366  oomph_info << "\n ERROR: Failed Mesh::self_test() for mesh imesh"
13367  << imesh << std::endl;
13368  }
13369  }
13370  }
13371 
13372 
13373  // Check global data
13374  unsigned Nglobal = Global_data_pt.size();
13375  for (unsigned iglobal = 0; iglobal < Nglobal; iglobal++)
13376  {
13377  if (Global_data_pt[iglobal]->self_test() != 0)
13378  {
13379  passed = false;
13380  oomph_info
13381  << "\n ERROR: Failed Data::self_test() for global data iglobal"
13382  << iglobal << std::endl;
13383  }
13384  }
13385 
13386 
13387 #ifdef OOMPH_HAS_MPI
13388 
13390  {
13391  // Note: This throws an error if it fails so no return is required.
13392  DocInfo tmp_doc_info;
13393  tmp_doc_info.disable_doc();
13394  check_halo_schemes(tmp_doc_info);
13395  }
13396 
13397 #endif
13398 
13399  // Return verdict
13400  if (passed)
13401  {
13402  return 0;
13403  }
13404  else
13405  {
13406  return 1;
13407  }
13408  }
13409 
13410  //====================================================================
13411  /// A function that is used to adapt a bifurcation-tracking
13412  /// problem, which requires separate interpolation of the
13413  /// associated eigenfunction. The error measure is chosen to be
13414  /// a suitable combination of the errors in the base flow and the
13415  /// eigenfunction. The bifurcation type is passed as an argument
13416  //=====================================================================
13417  void Problem::bifurcation_adapt_helper(unsigned& n_refined,
13418  unsigned& n_unrefined,
13419  const unsigned& bifurcation_type,
13420  const bool& actually_adapt)
13421  {
13422  // Storage for eigenfunction from the problem
13423  Vector<DoubleVector> eigenfunction;
13424  // Get the eigenfunction from the problem
13425  this->get_bifurcation_eigenfunction(eigenfunction);
13426 
13427  // Get the bifurcation parameter
13428  double* parameter_pt = this->bifurcation_parameter_pt();
13429 
13430  // Get the frequency parameter if tracking a Hopf bifurcation
13431  double omega = 0.0;
13432  // If we're tracking a Hopf then also get the frequency
13433  if (bifurcation_type == 3)
13434  {
13435  omega = dynamic_cast<HopfHandler*>(assembly_handler_pt())->omega();
13436  }
13437 
13438  // If we're tracking a Pitchfork get the slack parameter (Hack)
13439  double sigma = 0.0;
13440  if (bifurcation_type == 2)
13441  {
13442  sigma = this->dof(this->ndof() - 1);
13443  }
13444 
13445  // We can now deactivate the bifurcation tracking in the problem
13446  // to restore the degrees of freedom to the unaugmented value
13448 
13449  // Next, we create copies of the present problem
13450  // The number of copies depends on the number of eigenfunctions
13451  // One copy for each eigenfunction
13452  const unsigned n_copies = eigenfunction.size();
13453  Copy_of_problem_pt.resize(n_copies);
13454 
13455  // Loop over the number of copies
13456  for (unsigned c = 0; c < n_copies; c++)
13457  {
13458  // If we don't already have a copy
13459  if (Copy_of_problem_pt[c] == 0)
13460  {
13461  // Create the copy
13462  Copy_of_problem_pt[c] = this->make_copy();
13463 
13464  // Refine the copy to the same level as the current problem
13465 
13466  // Find number of submeshes
13467  const unsigned N_mesh = Copy_of_problem_pt[c]->nsub_mesh();
13468  // If there is only one mesh
13469  if (N_mesh == 0)
13470  {
13471  // Can we refine the mesh
13472  if (TreeBasedRefineableMeshBase* mmesh_pt =
13473  dynamic_cast<TreeBasedRefineableMeshBase*>(
13474  Copy_of_problem_pt[c]->mesh_pt(0)))
13475  {
13476  // Is the adapt flag set
13477  if (mmesh_pt->is_adaptation_enabled())
13478  {
13479  // Now get the original problem's mesh if it's refineable
13480  if (TreeBasedRefineableMeshBase* original_mesh_pt =
13481  dynamic_cast<TreeBasedRefineableMeshBase*>(
13482  this->mesh_pt(0)))
13483  {
13484  mmesh_pt->refine_base_mesh_as_in_reference_mesh(
13485  original_mesh_pt);
13486  }
13487  else
13488  {
13489  oomph_info
13490  << "Info/Warning: Mesh in orginal problem is not refineable."
13491  << std::endl;
13492  }
13493  }
13494  else
13495  {
13496  oomph_info << "Info/Warning: Mesh adaptation is disabled in copy."
13497  << std::endl;
13498  }
13499  }
13500  else
13501  {
13502  oomph_info << "Info/Warning: Mesh cannot be adapted in copy."
13503  << std::endl;
13504  }
13505  } // End of single mesh case
13506  // Otherwise loop over the submeshes
13507  else
13508  {
13509  for (unsigned m = 0; m < N_mesh; m++)
13510  {
13511  // Can we refine the submesh
13512  if (TreeBasedRefineableMeshBase* mmesh_pt =
13513  dynamic_cast<TreeBasedRefineableMeshBase*>(
13514  Copy_of_problem_pt[c]->mesh_pt(m)))
13515  {
13516  // Is the adapt flag set
13517  if (mmesh_pt->is_adaptation_enabled())
13518  {
13519  // Now get the original problem's mesh
13520  if (TreeBasedRefineableMeshBase* original_mesh_pt =
13521  dynamic_cast<TreeBasedRefineableMeshBase*>(
13522  this->mesh_pt(m)))
13523  {
13524  mmesh_pt->refine_base_mesh_as_in_reference_mesh(
13525  original_mesh_pt);
13526  }
13527  else
13528  {
13529  oomph_info << "Info/Warning: Mesh in orginal problem is not "
13530  "refineable."
13531  << std::endl;
13532  }
13533  }
13534  else
13535  {
13536  oomph_info
13537  << "Info/Warning: Mesh adaptation is disabled in copy."
13538  << std::endl;
13539  }
13540  }
13541  else
13542  {
13543  oomph_info << "Info/Warning: Mesh cannot be adapted in copy."
13544  << std::endl;
13545  }
13546  }
13547  // rebuild the global mesh in the copy
13548  Copy_of_problem_pt[c]->rebuild_global_mesh();
13549 
13550  } // End of multiple mesh case
13551 
13552  // Must call actions after adapt
13553  Copy_of_problem_pt[c]->actions_after_adapt();
13554 
13555  // Assign the equation numbers to the copy (quietly)
13557  }
13558  } // End of creation of copies
13559 
13560 
13561  // Now check some numbers
13562  for (unsigned c = 0; c < n_copies; c++)
13563  {
13564  // Check that the dofs match for each copy
13565 #ifdef PARANOID
13566  // If the problems don't match then complain
13567  if (Copy_of_problem_pt[c]->ndof() != this->ndof())
13568  {
13569  std::ostringstream error_stream;
13570  error_stream << "Number of unknowns in the problem copy " << c << " "
13571  << "not equal to number in the original:\n"
13572  << this->ndof() << " (original) "
13573  << Copy_of_problem_pt[c]->ndof() << " (copy)\n";
13574 
13575  throw OomphLibError(
13576  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
13577  }
13578 #endif
13579 
13580  // Assign the eigenfunction(s) to the copied problems
13581  Copy_of_problem_pt[c]->assign_eigenvector_to_dofs(eigenfunction[c]);
13582  // Set all pinned values to zero
13583  Copy_of_problem_pt[c]->set_pinned_values_to_zero();
13584  }
13585 
13586  // Symmetrise the problem if we are solving a pitchfork
13587  if (bifurcation_type == 2)
13588  {
13590  ->symmetrise_eigenfunction_for_adaptive_pitchfork_tracking();
13591  }
13592 
13593  // Find error estimates based on current problem and eigenproblem
13594  // Now we need to get the error estimates for both problems.
13595  Vector<Vector<double>> base_error, eigenfunction_error;
13596  this->get_all_error_estimates(base_error);
13597  // Loop over the copies
13598  for (unsigned c = 0; c < n_copies; c++)
13599  {
13600  // Get the error estimates for the copy
13601  Copy_of_problem_pt[c]->get_all_error_estimates(eigenfunction_error);
13602 
13603  // Find the number of meshes
13604  unsigned n_mesh = base_error.size();
13605 
13606 #ifdef PARANOID
13607  if (n_mesh != eigenfunction_error.size())
13608  {
13609  std::ostringstream error_stream;
13610  error_stream << "Problems do not have the same number of meshes\n"
13611  << "Base : " << n_mesh
13612  << " : Eigenproblem : " << eigenfunction_error.size()
13613  << "\n";
13614  throw OomphLibError(
13615  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
13616  }
13617 #endif
13618 
13619  for (unsigned m = 0; m < n_mesh; m++)
13620  {
13621  // Check the number of elements is the same
13622  unsigned n_element = base_error[m].size();
13623 #ifdef PARANOID
13624  if (n_element != eigenfunction_error[m].size())
13625  {
13626  std::ostringstream error_stream;
13627  error_stream << "Mesh " << m
13628  << " does not have the same number of elements in the "
13629  "two problems:\n"
13630  << "Base: " << n_element
13631  << " : Eigenproblem: " << eigenfunction_error[m].size()
13632  << "\n";
13633  throw OomphLibError(error_stream.str(),
13634  OOMPH_CURRENT_FUNCTION,
13635  OOMPH_EXCEPTION_LOCATION);
13636  }
13637 #endif
13638  // Now add all the error esimates together
13639  for (unsigned e = 0; e < n_element; e++)
13640  {
13641  // Add the error estimates (lazy)
13642  base_error[m][e] += eigenfunction_error[m][e];
13643  }
13644  }
13645  } // End of loop over copies
13646 
13647  // Then refine all problems based on the combined measure
13648  // if we are actually adapting (not just estimating the errors)
13649  if (actually_adapt)
13650  {
13651  this->adapt_based_on_error_estimates(n_refined, n_unrefined, base_error);
13652  for (unsigned c = 0; c < n_copies; c++)
13653  {
13654  Copy_of_problem_pt[c]->adapt_based_on_error_estimates(
13655  n_refined, n_unrefined, base_error);
13656  }
13657  // Symmetrise the problem (again) if we are solving for a pitchfork
13658  if (bifurcation_type == 2)
13659  {
13661  ->symmetrise_eigenfunction_for_adaptive_pitchfork_tracking();
13662  }
13663 
13664  // Now get the refined guess for the eigenvector
13665  for (unsigned c = 0; c < n_copies; c++)
13666  {
13667  Copy_of_problem_pt[c]->get_dofs(eigenfunction[c]);
13668  }
13669  }
13670 
13671  // Reactivate the tracking
13672  switch (bifurcation_type)
13673  {
13674  // Fold tracking
13675  case 1:
13676  this->activate_fold_tracking(parameter_pt);
13677  break;
13678 
13679  // Pitchfork
13680  case 2:
13681  this->activate_pitchfork_tracking(parameter_pt, eigenfunction[0]);
13682  // reset the slack parameter
13683  this->dof(this->ndof() - 1) = sigma;
13684  break;
13685 
13686  // Hopf
13687  case 3:
13688  this->activate_hopf_tracking(
13689  parameter_pt, omega, eigenfunction[0], eigenfunction[1]);
13690  break;
13691 
13692  default:
13693  std::ostringstream error_stream;
13694  error_stream << "Bifurcation type " << bifurcation_type
13695  << " not known\n"
13696  << "1: Fold, 2: Pitchfork, 3: Hopf\n";
13697  throw OomphLibError(
13698  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
13699  }
13700  }
13701 
13702 
13703  //====================================================================
13704  /// A function that is used to document the errors when
13705  /// adapting a bifurcation-tracking
13706  /// problem, which requires separate interpolation of the
13707  /// associated eigenfunction. The error measure is chosen to be
13708  /// a suitable combination of the errors in the base flow and the
13709  /// eigenfunction. The bifurcation type is passed as an argument
13710  //=====================================================================
13711  void Problem::bifurcation_adapt_doc_errors(const unsigned& bifurcation_type)
13712  {
13713  // Dummy arguments
13714  unsigned n_refined, n_unrefined;
13715  // Just call the bifurcation helper without actually adapting
13716  bifurcation_adapt_helper(n_refined, n_unrefined, bifurcation_type, false);
13717  }
13718 
13719 
13720  //========================================================================
13721  /// Adapt problem:
13722  /// Perform mesh adaptation for (all) refineable (sub)mesh(es),
13723  /// based on their own error estimates and the target errors specified
13724  /// in the mesh(es). Following mesh adaptation,
13725  /// update global mesh, and re-assign equation numbers.
13726  /// Return # of refined/unrefined elements. On return from this
13727  /// function, Problem can immediately be solved again.
13728  //======================================================================
13729  void Problem::adapt(unsigned& n_refined, unsigned& n_unrefined)
13730  {
13731  double t_start_total = 0.0;
13733  {
13734  t_start_total = TimingHelpers::timer();
13735  }
13736 
13737  // Get the bifurcation type
13738  int bifurcation_type = this->Assembly_handler_pt->bifurcation_type();
13739 
13740  bool continuation_problem = false;
13741 
13742  // If we have continuation data then we need to project that across to the
13743  // new mesh
13745  {
13746  if (Dof_derivative.size() != 0)
13747  {
13748  continuation_problem = true;
13749  }
13750  }
13751 
13752  // If we are tracking a bifurcation then call the bifurcation adapt function
13753  if (bifurcation_type != 0)
13754  {
13755  this->bifurcation_adapt_helper(n_refined, n_unrefined, bifurcation_type);
13756  // Return immediately
13757  return;
13758  }
13759 
13760  if (continuation_problem)
13761  {
13762  // Create a copy of the problem
13763  Copy_of_problem_pt.resize(2);
13764  // If we don't already have a copy
13765  for (unsigned c = 0; c < 2; c++)
13766  {
13767  if (Copy_of_problem_pt[c] == 0)
13768  {
13769  // Create the copy
13770  Copy_of_problem_pt[c] = this->make_copy();
13771 
13772  // Refine the copy to the same level as the current problem
13773  // Must call actions before adapt
13774  Copy_of_problem_pt[c]->actions_before_adapt();
13775 
13776  // Find number of submeshes
13777  const unsigned N_mesh = Copy_of_problem_pt[c]->nsub_mesh();
13778 
13779  // If there is only one mesh
13780  if (N_mesh == 0)
13781  {
13782  // Can we refine the mesh
13783  if (TreeBasedRefineableMeshBase* mmesh_pt =
13784  dynamic_cast<TreeBasedRefineableMeshBase*>(
13785  Copy_of_problem_pt[c]->mesh_pt(0)))
13786  {
13787  // Is the adapt flag set
13788  if (mmesh_pt->is_adaptation_enabled())
13789  {
13790  // Now get the original problem's mesh if it's refineable
13791  if (TreeBasedRefineableMeshBase* original_mesh_pt =
13792  dynamic_cast<TreeBasedRefineableMeshBase*>(
13793  this->mesh_pt(0)))
13794  {
13795  if (dynamic_cast<SolidMesh*>(original_mesh_pt) != 0)
13796  {
13797  oomph_info
13798  << "Info/Warning: Adaptive Continuation is broken in "
13799  << "SolidElement" << std::endl;
13800  }
13801  mmesh_pt->refine_base_mesh_as_in_reference_mesh(
13802  original_mesh_pt);
13803  }
13804  else
13805  {
13806  oomph_info << "Info/Warning: Mesh in orginal problem is not "
13807  "refineable."
13808  << std::endl;
13809  }
13810  }
13811  else
13812  {
13813  oomph_info
13814  << "Info/Warning: Mesh adaptation is disabled in copy."
13815  << std::endl;
13816  }
13817  }
13818  else if (TriangleMeshBase* tmesh_pt =
13819  dynamic_cast<TriangleMeshBase*>(
13820  Copy_of_problem_pt[c]->mesh_pt(0)))
13821  {
13822  if (TriangleMeshBase* original_mesh_pt =
13823  dynamic_cast<TriangleMeshBase*>(this->mesh_pt(0)))
13824  {
13825  if (dynamic_cast<SolidMesh*>(original_mesh_pt) != 0)
13826  {
13827  oomph_info
13828  << "Info/Warning: Adaptive Continuation is broken in "
13829  << "SolidElement" << std::endl;
13830  }
13831 
13832  // Remesh using the triangulateIO of the base mesh
13833  // Done via a file, so a bit hacky but this will be
13834  // superseded very soon
13835  std::ofstream tri_dump("triangle_mesh.dmp");
13836  original_mesh_pt->dump_triangulateio(tri_dump);
13837  tri_dump.close();
13838  std::ifstream tri_read("triangle_mesh.dmp");
13839  tmesh_pt->remesh_from_triangulateio(tri_read);
13840  tri_read.close();
13841 
13842 
13843  // Set the nodes to be at the same positions
13844  // as the original just in case the
13845  // triangulatio is out of sync with the real data
13846  const unsigned n_node = original_mesh_pt->nnode();
13847  for (unsigned n = 0; n < n_node; ++n)
13848  {
13849  Node* const nod_pt = original_mesh_pt->node_pt(n);
13850  Node* const new_node_pt = tmesh_pt->node_pt(n);
13851  unsigned n_dim = nod_pt->ndim();
13852  for (unsigned i = 0; i < n_dim; ++i)
13853  {
13854  new_node_pt->x(i) = nod_pt->x(i);
13855  }
13856  }
13857  }
13858  else
13859  {
13860  oomph_info
13861  << "Info/warning: Original Mesh is not TriangleBased\n"
13862  << "... but the copy is!" << std::endl;
13863  }
13864  }
13865  else
13866  {
13867  oomph_info << "Info/Warning: Mesh cannot be adapted in copy."
13868  << std::endl;
13869  }
13870  } // End of single mesh case
13871  // Otherwise loop over the submeshes
13872  else
13873  {
13874  for (unsigned m = 0; m < N_mesh; m++)
13875  {
13876  // Can we refine the submesh
13877  if (TreeBasedRefineableMeshBase* mmesh_pt =
13878  dynamic_cast<TreeBasedRefineableMeshBase*>(
13879  Copy_of_problem_pt[c]->mesh_pt(m)))
13880  {
13881  // Is the adapt flag set
13882  if (mmesh_pt->is_adaptation_enabled())
13883  {
13884  // Now get the original problem's mesh
13885  if (TreeBasedRefineableMeshBase* original_mesh_pt =
13886  dynamic_cast<TreeBasedRefineableMeshBase*>(
13887  this->mesh_pt(m)))
13888  {
13889  if (dynamic_cast<SolidMesh*>(original_mesh_pt) != 0)
13890  {
13891  oomph_info
13892  << "Info/Warning: Adaptive Continuation is broken in "
13893  << "SolidElement" << std::endl;
13894  }
13895 
13896  mmesh_pt->refine_base_mesh_as_in_reference_mesh(
13897  original_mesh_pt);
13898  }
13899  else
13900  {
13901  oomph_info << "Info/Warning: Mesh in orginal problem is "
13902  "not refineable."
13903  << std::endl;
13904  }
13905  }
13906  else
13907  {
13908  oomph_info
13909  << "Info/Warning: Mesh adaptation is disabled in copy."
13910  << std::endl;
13911  }
13912  }
13913  else if (TriangleMeshBase* tmesh_pt =
13914  dynamic_cast<TriangleMeshBase*>(
13915  Copy_of_problem_pt[c]->mesh_pt(m)))
13916  {
13917  if (TriangleMeshBase* original_mesh_pt =
13918  dynamic_cast<TriangleMeshBase*>(this->mesh_pt(m)))
13919  {
13920  if (dynamic_cast<SolidMesh*>(original_mesh_pt) != 0)
13921  {
13922  oomph_info
13923  << "Info/Warning: Adaptive Continuation is broken in "
13924  << "SolidElement" << std::endl;
13925  }
13926 
13927  // Remesh using the triangulateIO of the base mesh
13928  // Done via a file, so a bit hacky but this will be
13929  // superseded very soon
13930  std::ofstream tri_dump("triangle_mesh.dmp");
13931  original_mesh_pt->dump_triangulateio(tri_dump);
13932  tri_dump.close();
13933  std::ifstream tri_read("triangle_mesh.dmp");
13934  tmesh_pt->remesh_from_triangulateio(tri_read);
13935  tri_read.close();
13936 
13937  // Set the nodes to be at the same positions
13938  // as the original just in case the
13939  // triangulatio is out of sync with the real data
13940  const unsigned n_node = original_mesh_pt->nnode();
13941  for (unsigned n = 0; n < n_node; ++n)
13942  {
13943  Node* const nod_pt = original_mesh_pt->node_pt(n);
13944  Node* const new_node_pt = tmesh_pt->node_pt(n);
13945  unsigned n_dim = nod_pt->ndim();
13946  for (unsigned i = 0; i < n_dim; ++i)
13947  {
13948  new_node_pt->x(i) = nod_pt->x(i);
13949  }
13950  }
13951  }
13952  else
13953  {
13954  oomph_info
13955  << "Info/warning: Original Mesh is not TriangleBased\n"
13956  << "... but the copy is!" << std::endl;
13957  }
13958  }
13959  else
13960  {
13961  oomph_info << "Info/Warning: Mesh cannot be adapted in copy."
13962  << std::endl;
13963  }
13964  }
13965 
13966 
13967  // Must call actions after adapt
13968  Copy_of_problem_pt[c]->actions_after_adapt();
13969 
13970  // rebuild the global mesh in the copy
13971  Copy_of_problem_pt[c]->rebuild_global_mesh();
13972 
13973  } // End of multiple mesh case
13974 
13975  // Must call actions after adapt
13976  Copy_of_problem_pt[c]->actions_after_adapt();
13977 
13978  // Assign the equation numbers to the copy (quietly)
13980  }
13981 
13982  // Check that the dofs match for each copy
13983 #ifdef PARANOID
13984  // If the problems don't match then complain
13985  if (Copy_of_problem_pt[c]->ndof() != this->ndof())
13986  {
13987  std::ostringstream error_stream;
13988  error_stream << "Number of unknowns in the problem copy " << c << " "
13989  << "not equal to number in the original:\n"
13990  << this->ndof() << " (original) "
13991  << Copy_of_problem_pt[c]->ndof() << " (copy)\n";
13992 
13993  throw OomphLibError(error_stream.str(),
13994  OOMPH_CURRENT_FUNCTION,
13995  OOMPH_EXCEPTION_LOCATION);
13996  }
13997 #endif
13998  }
13999 
14000  // Need to set the Dof derivatives to the copied problem
14001  // Assign the eigenfunction(s) to the copied problems
14002  unsigned ndof_local = Dof_distribution_pt->nrow_local();
14003  for (unsigned i = 0; i < ndof_local; i++)
14004  {
14005  Copy_of_problem_pt[0]->dof(i) = this->dof_derivative(i);
14006  Copy_of_problem_pt[1]->dof(i) = this->dof_current(i);
14007  }
14008  // Set all pinned values to zero
14009  Copy_of_problem_pt[0]->set_pinned_values_to_zero();
14010  // Don't need to for the current dofs that are actuall the dofs
14011 
14012  // Now adapt
14013  Vector<Vector<double>> base_error;
14014  this->get_all_error_estimates(base_error);
14015  this->adapt_based_on_error_estimates(n_refined, n_unrefined, base_error);
14016  Copy_of_problem_pt[0]->adapt_based_on_error_estimates(
14017  n_refined, n_unrefined, base_error);
14018  Copy_of_problem_pt[1]->adapt_based_on_error_estimates(
14019  n_refined, n_unrefined, base_error);
14020 
14021  // Now sort out the Dof pointer
14022  ndof_local = Dof_distribution_pt->nrow_local();
14023  if (Dof_derivative.size() != ndof_local)
14024  {
14025  Dof_derivative.resize(ndof_local, 0.0);
14026  }
14027  if (Dof_current.size() != ndof_local)
14028  {
14029  Dof_current.resize(ndof_local, 0.0);
14030  }
14031  for (unsigned i = 0; i < ndof_local; i++)
14032  {
14033  Dof_derivative[i] = Copy_of_problem_pt[0]->dof(i);
14034  Dof_current[i] = Copy_of_problem_pt[1]->dof(i);
14035  }
14036  // Return immediately
14037  return;
14038  }
14039 
14040  oomph_info << std::endl << std::endl;
14041  oomph_info << "Adapting problem:" << std::endl;
14042  oomph_info << "=================" << std::endl;
14043 
14044  double t_start = 0.0;
14046  {
14047  t_start = TimingHelpers::timer();
14048  }
14049 
14050  // Call the actions before adaptation
14052 
14053  double t_end = 0.0;
14055  {
14056  t_end = TimingHelpers::timer();
14057  oomph_info << "Time for actions before adapt: " << t_end - t_start
14058  << std::endl;
14059  t_start = TimingHelpers::timer();
14060  }
14061 
14062  // Initialise counters
14063  n_refined = 0;
14064  n_unrefined = 0;
14065 
14066  // Number of submeshes?
14067  unsigned Nmesh = nsub_mesh();
14068 
14069  // Single mesh:
14070  //------------
14071  if (Nmesh == 0)
14072  {
14073  // Refine single mesh if possible
14074  if (RefineableMeshBase* mmesh_pt =
14075  dynamic_cast<RefineableMeshBase*>(mesh_pt(0)))
14076  {
14077  if (mmesh_pt->is_adaptation_enabled())
14078  {
14079  double t_start = TimingHelpers::timer();
14080 
14081  // Get pointer to error estimator
14082  ErrorEstimator* error_estimator_pt =
14083  mmesh_pt->spatial_error_estimator_pt();
14084 
14085 #ifdef PARANOID
14086  if (error_estimator_pt == 0)
14087  {
14088  throw OomphLibError("Error estimator hasn't been set yet",
14089  OOMPH_CURRENT_FUNCTION,
14090  OOMPH_EXCEPTION_LOCATION);
14091  }
14092 #endif
14093 
14094  // Get error for all elements
14095  Vector<double> elemental_error(mmesh_pt->nelement());
14096 
14097  if (mmesh_pt->doc_info_pt() == 0)
14098  {
14099  error_estimator_pt->get_element_errors(mesh_pt(0), elemental_error);
14100  }
14101  else
14102  {
14103  error_estimator_pt->get_element_errors(
14104  mesh_pt(0), elemental_error, *mmesh_pt->doc_info_pt());
14105  }
14106 
14107  // Store max./min actual error
14108  mmesh_pt->max_error() = std::fabs(*std::max_element(
14109  elemental_error.begin(), elemental_error.end(), AbsCmp<double>()));
14110 
14111  mmesh_pt->min_error() = std::fabs(*std::min_element(
14112  elemental_error.begin(), elemental_error.end(), AbsCmp<double>()));
14113 
14114  oomph_info << "\n Max/min error: " << mmesh_pt->max_error() << " "
14115  << mmesh_pt->min_error() << std::endl
14116  << std::endl;
14117 
14118 
14120  {
14121  t_end = TimingHelpers::timer();
14122  oomph_info << "Time for error estimation: " << t_end - t_start
14123  << std::endl;
14124  t_start = TimingHelpers::timer();
14125  }
14126 
14127  // Adapt mesh
14128  mmesh_pt->adapt(elemental_error);
14129 
14130  // Add to counters
14131  n_refined += mmesh_pt->nrefined();
14132  n_unrefined += mmesh_pt->nunrefined();
14133 
14135  {
14136  t_end = TimingHelpers::timer();
14137  oomph_info << "Time for complete mesh adaptation "
14138  << "(but excluding comp of error estimate): "
14139  << t_end - t_start << std::endl;
14140  t_start = TimingHelpers::timer();
14141  }
14142  }
14143  else
14144  {
14145  oomph_info << "Info/Warning: Mesh adaptation is disabled."
14146  << std::endl;
14147  }
14148  }
14149  else
14150  {
14151  oomph_info << "Info/Warning: Mesh cannot be adapted" << std::endl;
14152  }
14153  }
14154  // Multiple submeshes
14155  //------------------
14156  else
14157  {
14158  // Loop over submeshes
14159  for (unsigned imesh = 0; imesh < Nmesh; imesh++)
14160  {
14161  // Refine single mesh uniformly if possible
14162  if (RefineableMeshBase* mmesh_pt =
14163  dynamic_cast<RefineableMeshBase*>(mesh_pt(imesh)))
14164  {
14165  double t_start = TimingHelpers::timer();
14166 
14167  // Get pointer to error estimator
14168  ErrorEstimator* error_estimator_pt =
14169  mmesh_pt->spatial_error_estimator_pt();
14170 
14171 #ifdef PARANOID
14172  if (error_estimator_pt == 0)
14173  {
14174  throw OomphLibError("Error estimator hasn't been set yet",
14175  OOMPH_CURRENT_FUNCTION,
14176  OOMPH_EXCEPTION_LOCATION);
14177  }
14178 #endif
14179 
14180  if (mmesh_pt->is_adaptation_enabled())
14181  {
14182  // Get error for all elements
14183  Vector<double> elemental_error(mmesh_pt->nelement());
14184  if (mmesh_pt->doc_info_pt() == 0)
14185  {
14186  error_estimator_pt->get_element_errors(mesh_pt(imesh),
14187  elemental_error);
14188  }
14189  else
14190  {
14191  error_estimator_pt->get_element_errors(
14192  mesh_pt(imesh), elemental_error, *mmesh_pt->doc_info_pt());
14193  }
14194 
14195  // Store max./min error if the mesh has any elements
14196  if (mesh_pt(imesh)->nelement() > 0)
14197  {
14198  mmesh_pt->max_error() =
14199  std::fabs(*std::max_element(elemental_error.begin(),
14200  elemental_error.end(),
14201  AbsCmp<double>()));
14202 
14203  mmesh_pt->min_error() =
14204  std::fabs(*std::min_element(elemental_error.begin(),
14205  elemental_error.end(),
14206  AbsCmp<double>()));
14207  }
14208 
14209  oomph_info << "\n Max/min error: " << mmesh_pt->max_error() << " "
14210  << mmesh_pt->min_error() << std::endl;
14211 
14212 
14214  {
14215  t_end = TimingHelpers::timer();
14216  oomph_info << "Time for error estimation: " << t_end - t_start
14217  << std::endl;
14218  t_start = TimingHelpers::timer();
14219  }
14220 
14221  // Adapt mesh
14222  mmesh_pt->adapt(elemental_error);
14223 
14224  // Add to counters
14225  n_refined += mmesh_pt->nrefined();
14226  n_unrefined += mmesh_pt->nunrefined();
14227 
14228 
14230  {
14231  t_end = TimingHelpers::timer();
14232  oomph_info << "Time for complete mesh adaptation "
14233  << "(but excluding comp of error estimate): "
14234  << t_end - t_start << std::endl;
14235  t_start = TimingHelpers::timer();
14236  }
14237  }
14238  else
14239  {
14240  oomph_info << "Info/Warning: Mesh adaptation is disabled."
14241  << std::endl;
14242  }
14243  }
14244  else
14245  {
14246  oomph_info << "Info/Warning: Mesh cannot be adapted." << std::endl;
14247  }
14248 
14249  } // End of loop over submeshes
14250 
14251  // Rebuild the global mesh
14253  }
14254 
14255 
14257  {
14258  t_end = TimingHelpers::timer();
14259  oomph_info << "Total time for actual adaptation "
14260  << "(all meshes; incl error estimates): " << t_end - t_start
14261  << std::endl;
14262  t_start = TimingHelpers::timer();
14263  }
14264 
14265  // Any actions after adapt
14267 
14268 
14270  {
14271  t_end = TimingHelpers::timer();
14272  oomph_info << "Time for actions after adapt: " << t_end - t_start
14273  << std::endl;
14274  t_start = TimingHelpers::timer();
14275 
14276  oomph_info << "About to start re-assigning eqn numbers "
14277  << "with Problem::assign_eqn_numbers() at end of "
14278  << "Problem::adapt().\n";
14279  }
14280 
14281  // Attach the boundary conditions to the mesh
14282  oomph_info << "\nNumber of equations: " << assign_eqn_numbers() << std::endl
14283  << std::endl;
14284 
14285 
14287  {
14288  t_end = TimingHelpers::timer();
14289  oomph_info << "Time for re-assigning eqn numbers with "
14290  << "Problem::assign_eqn_numbers() at end of Problem::adapt(): "
14291  << t_end - t_start << std::endl;
14292  oomph_info << "Total time for adapt: " << t_end - t_start_total
14293  << std::endl;
14294  }
14295  }
14296 
14297  //========================================================================
14298  /// p-adapt problem:
14299  /// Perform mesh adaptation for (all) refineable (sub)mesh(es),
14300  /// based on their own error estimates and the target errors specified
14301  /// in the mesh(es). Following mesh adaptation,
14302  /// update global mesh, and re-assign equation numbers.
14303  /// Return # of refined/unrefined elements. On return from this
14304  /// function, Problem can immediately be solved again.
14305  //======================================================================
14306  void Problem::p_adapt(unsigned& n_refined, unsigned& n_unrefined)
14307  {
14308  double t_start_total = 0.0;
14310  {
14311  t_start_total = TimingHelpers::timer();
14312  }
14313 
14314  // Get the bifurcation type
14315  int bifurcation_type = this->Assembly_handler_pt->bifurcation_type();
14316 
14317  // If we are tracking a bifurcation then call the bifurcation adapt function
14318  if (bifurcation_type != 0)
14319  {
14320  this->bifurcation_adapt_helper(n_refined, n_unrefined, bifurcation_type);
14321  // Return immediately
14322  return;
14323  }
14324 
14325  oomph_info << std::endl << std::endl;
14326  oomph_info << "p-adapting problem:" << std::endl;
14327  oomph_info << "===================" << std::endl;
14328 
14329  double t_start = 0.0;
14331  {
14332  t_start = TimingHelpers::timer();
14333  }
14334 
14335  // Call the actions before adaptation
14337 
14338  double t_end = 0.0;
14340  {
14341  t_end = TimingHelpers::timer();
14342  oomph_info << "Time for actions before adapt: " << t_end - t_start
14343  << std::endl;
14344  t_start = TimingHelpers::timer();
14345  }
14346 
14347  // Initialise counters
14348  n_refined = 0;
14349  n_unrefined = 0;
14350 
14351  // Number of submeshes?
14352  unsigned Nmesh = nsub_mesh();
14353 
14354  // Single mesh:
14355  //------------
14356  if (Nmesh == 0)
14357  {
14358  // Refine single mesh if possible
14359  if (RefineableMeshBase* mmesh_pt =
14360  dynamic_cast<RefineableMeshBase*>(mesh_pt(0)))
14361  {
14362  if (mmesh_pt->is_p_adaptation_enabled())
14363  {
14364  double t_start = TimingHelpers::timer();
14365 
14366  // Get pointer to error estimator
14367  ErrorEstimator* error_estimator_pt =
14368  mmesh_pt->spatial_error_estimator_pt();
14369 
14370 #ifdef PARANOID
14371  if (error_estimator_pt == 0)
14372  {
14373  throw OomphLibError("Error estimator hasn't been set yet",
14374  OOMPH_CURRENT_FUNCTION,
14375  OOMPH_EXCEPTION_LOCATION);
14376  }
14377 #endif
14378 
14379  // Get error for all elements
14380  Vector<double> elemental_error(mmesh_pt->nelement());
14381 
14382  if (mmesh_pt->doc_info_pt() == 0)
14383  {
14384  error_estimator_pt->get_element_errors(mesh_pt(0), elemental_error);
14385  }
14386  else
14387  {
14388  error_estimator_pt->get_element_errors(
14389  mesh_pt(0), elemental_error, *mmesh_pt->doc_info_pt());
14390  }
14391 
14392  // Store max./min actual error
14393  mmesh_pt->max_error() = std::fabs(*std::max_element(
14394  elemental_error.begin(), elemental_error.end(), AbsCmp<double>()));
14395 
14396  mmesh_pt->min_error() = std::fabs(*std::min_element(
14397  elemental_error.begin(), elemental_error.end(), AbsCmp<double>()));
14398 
14399  oomph_info << "\n Max/min error: " << mmesh_pt->max_error() << " "
14400  << mmesh_pt->min_error() << std::endl
14401  << std::endl;
14402 
14403 
14405  {
14406  t_end = TimingHelpers::timer();
14407  oomph_info << "Time for error estimation: " << t_end - t_start
14408  << std::endl;
14409  t_start = TimingHelpers::timer();
14410  }
14411 
14412  // Adapt mesh
14413  mmesh_pt->p_adapt(elemental_error);
14414 
14415  // Add to counters
14416  n_refined += mmesh_pt->nrefined();
14417  n_unrefined += mmesh_pt->nunrefined();
14418 
14420  {
14421  t_end = TimingHelpers::timer();
14422  oomph_info << "Time for complete mesh adaptation "
14423  << "(but excluding comp of error estimate): "
14424  << t_end - t_start << std::endl;
14425  t_start = TimingHelpers::timer();
14426  }
14427  }
14428  else
14429  {
14430  oomph_info << "Info/Warning: Mesh adaptation is disabled."
14431  << std::endl;
14432  }
14433  }
14434  else
14435  {
14436  oomph_info << "Info/Warning: Mesh cannot be adapted" << std::endl;
14437  }
14438  }
14439  // Multiple submeshes
14440  //------------------
14441  else
14442  {
14443  // Loop over submeshes
14444  for (unsigned imesh = 0; imesh < Nmesh; imesh++)
14445  {
14446  // Refine single mesh uniformly if possible
14447  if (RefineableMeshBase* mmesh_pt =
14448  dynamic_cast<RefineableMeshBase*>(mesh_pt(imesh)))
14449  {
14450  double t_start = TimingHelpers::timer();
14451 
14452  // Get pointer to error estimator
14453  ErrorEstimator* error_estimator_pt =
14454  mmesh_pt->spatial_error_estimator_pt();
14455 
14456 #ifdef PARANOID
14457  if (error_estimator_pt == 0)
14458  {
14459  throw OomphLibError("Error estimator hasn't been set yet",
14460  OOMPH_CURRENT_FUNCTION,
14461  OOMPH_EXCEPTION_LOCATION);
14462  }
14463 #endif
14464 
14465  if (mmesh_pt->is_p_adaptation_enabled())
14466  {
14467  // Get error for all elements
14468  Vector<double> elemental_error(mmesh_pt->nelement());
14469  if (mmesh_pt->doc_info_pt() == 0)
14470  {
14471  error_estimator_pt->get_element_errors(mesh_pt(imesh),
14472  elemental_error);
14473  }
14474  else
14475  {
14476  error_estimator_pt->get_element_errors(
14477  mesh_pt(imesh), elemental_error, *mmesh_pt->doc_info_pt());
14478  }
14479 
14480  // Store max./min error if the mesh has any elements
14481  if (mesh_pt(imesh)->nelement() > 0)
14482  {
14483  mmesh_pt->max_error() =
14484  std::fabs(*std::max_element(elemental_error.begin(),
14485  elemental_error.end(),
14486  AbsCmp<double>()));
14487 
14488  mmesh_pt->min_error() =
14489  std::fabs(*std::min_element(elemental_error.begin(),
14490  elemental_error.end(),
14491  AbsCmp<double>()));
14492  }
14493 
14494  oomph_info << "\n Max/min error: " << mmesh_pt->max_error() << " "
14495  << mmesh_pt->min_error() << std::endl;
14496 
14497 
14499  {
14500  t_end = TimingHelpers::timer();
14501  oomph_info << "Time for error estimation: " << t_end - t_start
14502  << std::endl;
14503  t_start = TimingHelpers::timer();
14504  }
14505 
14506  // Adapt mesh
14507  mmesh_pt->p_adapt(elemental_error);
14508 
14509  // Add to counters
14510  n_refined += mmesh_pt->nrefined();
14511  n_unrefined += mmesh_pt->nunrefined();
14512 
14513 
14515  {
14516  t_end = TimingHelpers::timer();
14517  oomph_info << "Time for complete mesh adaptation "
14518  << "(but excluding comp of error estimate): "
14519  << t_end - t_start << std::endl;
14520  t_start = TimingHelpers::timer();
14521  }
14522  }
14523  else
14524  {
14525  oomph_info << "Info/Warning: Mesh adaptation is disabled."
14526  << std::endl;
14527  }
14528  }
14529  else
14530  {
14531  oomph_info << "Info/Warning: Mesh cannot be adapted." << std::endl;
14532  }
14533 
14534  } // End of loop over submeshes
14535 
14536  // Rebuild the global mesh
14538  }
14539 
14540 
14542  {
14543  t_end = TimingHelpers::timer();
14544  oomph_info << "Total time for actual adaptation "
14545  << "(all meshes; incl error estimates): " << t_end - t_start
14546  << std::endl;
14547  t_start = TimingHelpers::timer();
14548  }
14549 
14550  // Any actions after adapt
14552 
14553 
14555  {
14556  t_end = TimingHelpers::timer();
14557  oomph_info << "Time for actions after adapt: " << t_end - t_start
14558  << std::endl;
14559  t_start = TimingHelpers::timer();
14560 
14561  oomph_info << "About to start re-assigning eqn numbers "
14562  << "with Problem::assign_eqn_numbers() at end of "
14563  << "Problem::adapt().\n";
14564  }
14565 
14566  // Attach the boundary conditions to the mesh
14567  oomph_info << "\nNumber of equations: " << assign_eqn_numbers() << std::endl
14568  << std::endl;
14569 
14570 
14572  {
14573  t_end = TimingHelpers::timer();
14574  oomph_info << "Time for re-assigning eqn numbers with "
14575  << "Problem::assign_eqn_numbers() at end of Problem::adapt(): "
14576  << t_end - t_start << std::endl;
14577  oomph_info << "Total time for adapt: " << t_end - t_start_total
14578  << std::endl;
14579  }
14580  }
14581 
14582  //========================================================================
14583  /// Perform mesh adaptation for (all) refineable (sub)mesh(es),
14584  /// based on the error estimates in elemental_error
14585  /// and the target errors specified
14586  /// in the mesh(es). Following mesh adaptation,
14587  /// update global mesh, and re-assign equation numbers.
14588  /// Return # of refined/unrefined elements. On return from this
14589  /// function, Problem can immediately be solved again.
14590  //========================================================================
14592  unsigned& n_refined,
14593  unsigned& n_unrefined,
14594  Vector<Vector<double>>& elemental_error)
14595  {
14596  oomph_info << std::endl << std::endl;
14597  oomph_info << "Adapting problem:" << std::endl;
14598  oomph_info << "=================" << std::endl;
14599 
14600  // Call the actions before adaptation
14602 
14603  // Initialise counters
14604  n_refined = 0;
14605  n_unrefined = 0;
14606 
14607  // Number of submeshes?
14608  unsigned Nmesh = nsub_mesh();
14609 
14610  // Single mesh:
14611  //------------
14612  if (Nmesh == 0)
14613  {
14614  // Refine single mesh uniformly if possible
14615  if (RefineableMeshBase* mmesh_pt =
14616  dynamic_cast<RefineableMeshBase*>(Problem::mesh_pt(0)))
14617  {
14618  if (mmesh_pt->is_adaptation_enabled())
14619  {
14620  // Adapt mesh
14621  mmesh_pt->adapt(elemental_error[0]);
14622 
14623  // Add to counters
14624  n_refined += mmesh_pt->nrefined();
14625  n_unrefined += mmesh_pt->nunrefined();
14626  }
14627  else
14628  {
14629  oomph_info << "Info/Warning: Mesh adaptation is disabled."
14630  << std::endl;
14631  }
14632  }
14633  else
14634  {
14635  oomph_info << "Info/Warning: Mesh cannot be adapted" << std::endl;
14636  }
14637  }
14638 
14639  // Multiple submeshes
14640  //------------------
14641  else
14642  {
14643  // Loop over submeshes
14644  for (unsigned imesh = 0; imesh < Nmesh; imesh++)
14645  {
14646  // Refine single mesh uniformly if possible
14647  if (RefineableMeshBase* mmesh_pt =
14648  dynamic_cast<RefineableMeshBase*>(Problem::mesh_pt(imesh)))
14649  {
14650  if (mmesh_pt->is_adaptation_enabled())
14651  {
14652  // Adapt mesh
14653  mmesh_pt->adapt(elemental_error[imesh]);
14654 
14655  // Add to counters
14656  n_refined += mmesh_pt->nrefined();
14657  n_unrefined += mmesh_pt->nunrefined();
14658  }
14659  else
14660  {
14661  oomph_info << "Info/Warning: Mesh adaptation is disabled."
14662  << std::endl;
14663  }
14664  }
14665  else
14666  {
14667  oomph_info << "Info/Warning: Mesh cannot be adapted." << std::endl;
14668  }
14669 
14670  } // End of loop over submeshes
14671 
14672  // Rebuild the global mesh
14674  }
14675 
14676  // Any actions after adapt
14678 
14679  // Attach the boundary conditions to the mesh
14680  oomph_info << "\nNumber of equations: " << assign_eqn_numbers() << std::endl
14681  << std::endl;
14682  }
14683 
14684 
14685  //========================================================================
14686  /// Return the error estimates computed by (all) refineable
14687  /// (sub)mesh(es) in the elemental_error structure, which consists of
14688  /// a vector of elemental errors for each (sub)mesh.
14689  //========================================================================
14691  {
14692  // Number of submeshes?
14693  const unsigned Nmesh = nsub_mesh();
14694 
14695  // Single mesh:
14696  //------------
14697  if (Nmesh == 0)
14698  {
14699  // There is only one mesh
14700  elemental_error.resize(1);
14701  // Refine single mesh uniformly if possible
14702  if (RefineableMeshBase* mmesh_pt =
14703  dynamic_cast<RefineableMeshBase*>(Problem::mesh_pt(0)))
14704  {
14705  // If we can adapt the mesh
14706  if (mmesh_pt->is_adaptation_enabled())
14707  {
14708  // Get pointer to error estimator
14709  ErrorEstimator* error_estimator_pt =
14710  mmesh_pt->spatial_error_estimator_pt();
14711 
14712 #ifdef PARANOID
14713  if (error_estimator_pt == 0)
14714  {
14715  throw OomphLibError("Error estimator hasn't been set yet",
14716  OOMPH_CURRENT_FUNCTION,
14717  OOMPH_EXCEPTION_LOCATION);
14718  }
14719 #endif
14720 
14721  // Get error for all elements
14722  elemental_error[0].resize(mmesh_pt->nelement());
14723  // Are we documenting the errors or not
14724  if (mmesh_pt->doc_info_pt() == 0)
14725  {
14726  error_estimator_pt->get_element_errors(Problem::mesh_pt(0),
14727  elemental_error[0]);
14728  }
14729  else
14730  {
14731  error_estimator_pt->get_element_errors(Problem::mesh_pt(0),
14732  elemental_error[0],
14733  *mmesh_pt->doc_info_pt());
14734  }
14735 
14736  // Store max./min actual error
14737  mmesh_pt->max_error() =
14738  std::fabs(*std::max_element(elemental_error[0].begin(),
14739  elemental_error[0].end(),
14740  AbsCmp<double>()));
14741 
14742  mmesh_pt->min_error() =
14743  std::fabs(*std::min_element(elemental_error[0].begin(),
14744  elemental_error[0].end(),
14745  AbsCmp<double>()));
14746 
14747  oomph_info << "\n Max/min error: " << mmesh_pt->max_error() << " "
14748  << mmesh_pt->min_error() << std::endl;
14749  }
14750  else
14751  {
14752  oomph_info << "Info/Warning: Mesh adaptation is disabled."
14753  << std::endl;
14754  }
14755  }
14756  else
14757  {
14758  oomph_info << "Info/Warning: Mesh cannot be adapted" << std::endl;
14759  }
14760  }
14761 
14762  // Multiple submeshes
14763  //------------------
14764  else
14765  {
14766  // Resize to the number of submeshes
14767  elemental_error.resize(Nmesh);
14768 
14769  // Loop over submeshes
14770  for (unsigned imesh = 0; imesh < Nmesh; imesh++)
14771  {
14772  // Refine single mesh uniformly if possible
14773  if (RefineableMeshBase* mmesh_pt =
14774  dynamic_cast<RefineableMeshBase*>(Problem::mesh_pt(imesh)))
14775  {
14776  // Get pointer to error estimator
14777  ErrorEstimator* error_estimator_pt =
14778  mmesh_pt->spatial_error_estimator_pt();
14779 
14780 #ifdef PARANOID
14781  if (error_estimator_pt == 0)
14782  {
14783  throw OomphLibError("Error estimator hasn't been set yet",
14784  OOMPH_CURRENT_FUNCTION,
14785  OOMPH_EXCEPTION_LOCATION);
14786  }
14787 #endif
14788  // If we can adapt the mesh
14789  if (mmesh_pt->is_adaptation_enabled())
14790  {
14791  // Get error for all elements
14792  elemental_error[imesh].resize(mmesh_pt->nelement());
14793  if (mmesh_pt->doc_info_pt() == 0)
14794  {
14795  error_estimator_pt->get_element_errors(Problem::mesh_pt(imesh),
14796  elemental_error[imesh]);
14797  }
14798  else
14799  {
14800  error_estimator_pt->get_element_errors(Problem::mesh_pt(imesh),
14801  elemental_error[imesh],
14802  *mmesh_pt->doc_info_pt());
14803  }
14804 
14805  // Store max./min error
14806  mmesh_pt->max_error() =
14807  std::fabs(*std::max_element(elemental_error[imesh].begin(),
14808  elemental_error[imesh].end(),
14809  AbsCmp<double>()));
14810 
14811  mmesh_pt->min_error() =
14812  std::fabs(*std::min_element(elemental_error[imesh].begin(),
14813  elemental_error[imesh].end(),
14814  AbsCmp<double>()));
14815 
14816  oomph_info << "\n Max/min error: " << mmesh_pt->max_error() << " "
14817  << mmesh_pt->min_error() << std::endl;
14818  }
14819  else
14820  {
14821  oomph_info << "Info/Warning: Mesh adaptation is disabled."
14822  << std::endl;
14823  }
14824  }
14825  else
14826  {
14827  oomph_info << "Info/Warning: Mesh cannot be adapted." << std::endl;
14828  }
14829 
14830  } // End of loop over submeshes
14831  }
14832  }
14833 
14834  //========================================================================
14835  /// Get max and min error for all elements in submeshes
14836  //========================================================================
14838  {
14839  // Get the bifurcation type
14840  int bifurcation_type = this->Assembly_handler_pt->bifurcation_type();
14841  // If we are tracking a bifurcation then call the bifurcation adapt function
14842  if (bifurcation_type != 0)
14843  {
14844  this->bifurcation_adapt_doc_errors(bifurcation_type);
14845  // Return immediately
14846  return;
14847  }
14848 
14849  // Number of submeshes?
14850  unsigned Nmesh = nsub_mesh();
14851 
14852  // Single mesh:
14853  //------------
14854  if (Nmesh == 0)
14855  {
14856  // Is the single mesh refineable?
14857  if (RefineableMeshBase* mmesh_pt =
14858  dynamic_cast<RefineableMeshBase*>(mesh_pt(0)))
14859  {
14860  // Get pointer to error estimator
14861  ErrorEstimator* error_estimator_pt =
14862  mmesh_pt->spatial_error_estimator_pt();
14863 
14864 #ifdef PARANOID
14865  if (error_estimator_pt == 0)
14866  {
14867  throw OomphLibError("Error estimator hasn't been set yet",
14868  OOMPH_CURRENT_FUNCTION,
14869  OOMPH_EXCEPTION_LOCATION);
14870  }
14871 #endif
14872 
14873  // Get error for all elements
14874  Vector<double> elemental_error(mmesh_pt->nelement());
14875  if (!doc_info.is_doc_enabled())
14876  {
14877  error_estimator_pt->get_element_errors(mesh_pt(0), elemental_error);
14878  }
14879  else
14880  {
14881  error_estimator_pt->get_element_errors(
14882  mesh_pt(0), elemental_error, doc_info);
14883  }
14884 
14885  // Store max./min actual error
14886  mmesh_pt->max_error() = std::fabs(*std::max_element(
14887  elemental_error.begin(), elemental_error.end(), AbsCmp<double>()));
14888 
14889  mmesh_pt->min_error() = std::fabs(*std::min_element(
14890  elemental_error.begin(), elemental_error.end(), AbsCmp<double>()));
14891 
14892  oomph_info << "\n Max/min error: " << mmesh_pt->max_error() << " "
14893  << mmesh_pt->min_error() << std::endl;
14894  }
14895  }
14896 
14897  // Multiple submeshes
14898  //------------------
14899  else
14900  {
14901  // Loop over submeshes
14902  for (unsigned imesh = 0; imesh < Nmesh; imesh++)
14903  {
14904  // Is the single mesh refineable?
14905  if (RefineableMeshBase* mmesh_pt =
14906  dynamic_cast<RefineableMeshBase*>(mesh_pt(imesh)))
14907  {
14908  // Get pointer to error estimator
14909  ErrorEstimator* error_estimator_pt =
14910  mmesh_pt->spatial_error_estimator_pt();
14911 
14912 #ifdef PARANOID
14913  if (error_estimator_pt == 0)
14914  {
14915  throw OomphLibError("Error estimator hasn't been set yet",
14916  OOMPH_CURRENT_FUNCTION,
14917  OOMPH_EXCEPTION_LOCATION);
14918  }
14919 #endif
14920 
14921  // Get error for all elements
14922  Vector<double> elemental_error(mmesh_pt->nelement());
14923  if (mmesh_pt->doc_info_pt() == 0)
14924  {
14925  error_estimator_pt->get_element_errors(mesh_pt(imesh),
14926  elemental_error);
14927  }
14928  else
14929  {
14930  error_estimator_pt->get_element_errors(
14931  mesh_pt(imesh), elemental_error, *mmesh_pt->doc_info_pt());
14932  }
14933 
14934  // Store max./min error if the mesh has any elements
14935  if (mesh_pt(imesh)->nelement() > 0)
14936  {
14937  mmesh_pt->max_error() =
14938  std::fabs(*std::max_element(elemental_error.begin(),
14939  elemental_error.end(),
14940  AbsCmp<double>()));
14941 
14942  mmesh_pt->min_error() =
14943  std::fabs(*std::min_element(elemental_error.begin(),
14944  elemental_error.end(),
14945  AbsCmp<double>()));
14946  }
14947 
14948  oomph_info << "\n Max/min error: " << mmesh_pt->max_error() << " "
14949  << mmesh_pt->min_error() << std::endl;
14950  }
14951 
14952  } // End of loop over submeshes
14953  }
14954  }
14955 
14956  //========================================================================
14957  /// Refine (one and only!) mesh by splitting the elements identified
14958  /// by their numbers relative to the problems' only mesh, then rebuild
14959  /// the problem.
14960  //========================================================================
14962  const Vector<unsigned>& elements_to_be_refined)
14963  {
14965 
14966  // Number of submeshes?
14967  unsigned Nmesh = nsub_mesh();
14968 
14969  // Single mesh:
14970  if (Nmesh == 0)
14971  {
14972  // Refine single mesh if possible
14973  if (TreeBasedRefineableMeshBase* mmesh_pt =
14974  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(0)))
14975  {
14976  mmesh_pt->refine_selected_elements(elements_to_be_refined);
14977  }
14978  else
14979  {
14980  oomph_info << "Info/Warning: Mesh cannot be refined " << std::endl;
14981  }
14982  }
14983  // Multiple submeshes
14984  else
14985  {
14986  std::ostringstream error_message;
14987  error_message << "Problem::refine_selected_elements(...) only works for\n"
14988  << "multiple-mesh problems if you specify the mesh\n"
14989  << "number in the function argument before the Vector,\n"
14990  << "or a Vector of Vectors for each submesh.\n"
14991  << std::endl;
14992  throw OomphLibError(
14993  error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
14994  }
14995 
14996  // Any actions after the adapatation phase
14998 
14999  // Attach the boundary conditions to the mesh
15000  oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
15001  }
15002 
15003  //========================================================================
15004  /// Refine (one and only!) mesh by splitting the elements identified
15005  /// by their pointers, then rebuild the problem.
15006  //========================================================================
15008  const Vector<RefineableElement*>& elements_to_be_refined_pt)
15009  {
15011 
15012  // Number of submeshes?
15013  unsigned Nmesh = nsub_mesh();
15014 
15015  // Single mesh:
15016  if (Nmesh == 0)
15017  {
15018  // Refine single mesh if possible
15019  if (TreeBasedRefineableMeshBase* mmesh_pt =
15020  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(0)))
15021  {
15022  mmesh_pt->refine_selected_elements(elements_to_be_refined_pt);
15023  }
15024  else
15025  {
15026  oomph_info << "Info/Warning: Mesh cannot be refined " << std::endl;
15027  }
15028  }
15029  // Multiple submeshes
15030  else
15031  {
15032  std::ostringstream error_message;
15033  error_message << "Problem::refine_selected_elements(...) only works for\n"
15034  << "multiple-mesh problems if you specify the mesh\n"
15035  << "number in the function argument before the Vector,\n"
15036  << "or a Vector of Vectors for each submesh.\n"
15037  << std::endl;
15038  throw OomphLibError(
15039  error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
15040  }
15041 
15042  // Any actions after the adapatation phase
15044 
15045  // Do equation numbering
15046  oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
15047  }
15048 
15049  //========================================================================
15050  /// Refine specified submesh by splitting the elements identified
15051  /// by their numbers relative to the specified mesh, then rebuild the problem.
15052  //========================================================================
15054  const unsigned& i_mesh, const Vector<unsigned>& elements_to_be_refined)
15055  {
15057 
15058  // Number of submeshes?
15059  unsigned n_mesh = nsub_mesh();
15060 
15061  if (i_mesh >= n_mesh)
15062  {
15063  std::ostringstream error_message;
15064  error_message << "Problem only has " << n_mesh
15065  << " submeshes. Cannot refine submesh " << i_mesh
15066  << std::endl;
15067  throw OomphLibError(
15068  error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
15069  }
15070 
15071  // Refine single mesh if possible
15072  if (TreeBasedRefineableMeshBase* mmesh_pt =
15073  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(i_mesh)))
15074  {
15075  mmesh_pt->refine_selected_elements(elements_to_be_refined);
15076  }
15077  else
15078  {
15079  oomph_info << "Info/Warning: Mesh cannot be refined " << std::endl;
15080  }
15081 
15082  if (n_mesh > 1)
15083  {
15084  // Rebuild the global mesh
15086  }
15087 
15088  // Any actions after the adapatation phase
15090 
15091  // Do equation numbering
15092  oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
15093  }
15094 
15095 
15096  //========================================================================
15097  /// Refine specified submesh by splitting the elements identified
15098  /// by their pointers, then rebuild the problem.
15099  //========================================================================
15101  const unsigned& i_mesh,
15102  const Vector<RefineableElement*>& elements_to_be_refined_pt)
15103  {
15105 
15106  // Number of submeshes?
15107  unsigned n_mesh = nsub_mesh();
15108 
15109  if (i_mesh >= n_mesh)
15110  {
15111  std::ostringstream error_message;
15112  error_message << "Problem only has " << n_mesh
15113  << " submeshes. Cannot refine submesh " << i_mesh
15114  << std::endl;
15115  throw OomphLibError(
15116  error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
15117  }
15118 
15119  // Refine single mesh if possible
15120  if (TreeBasedRefineableMeshBase* mmesh_pt =
15121  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(i_mesh)))
15122  {
15123  mmesh_pt->refine_selected_elements(elements_to_be_refined_pt);
15124  }
15125  else
15126  {
15127  oomph_info << "Info/Warning: Mesh cannot be refined " << std::endl;
15128  }
15129 
15130  if (n_mesh > 1)
15131  {
15132  // Rebuild the global mesh
15134  }
15135 
15136  // Any actions after the adapatation phase
15138 
15139  // Do equation numbering
15140  oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
15141  }
15142 
15143  //========================================================================
15144  /// Refine all submeshes by splitting the elements identified by their
15145  /// numbers relative to each submesh in a Vector of Vectors, then
15146  /// rebuild the problem.
15147  //========================================================================
15149  const Vector<Vector<unsigned>>& elements_to_be_refined)
15150  {
15152 
15153  // Number of submeshes?
15154  unsigned n_mesh = nsub_mesh();
15155 
15156  // Refine all submeshes if possible
15157  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
15158  {
15159  if (TreeBasedRefineableMeshBase* mmesh_pt =
15160  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(i_mesh)))
15161  {
15162  mmesh_pt->refine_selected_elements(elements_to_be_refined[i_mesh]);
15163  }
15164  else
15165  {
15166  oomph_info << "Info/Warning: Mesh cannot be refined " << std::endl;
15167  }
15168  }
15169 
15170  // Rebuild the global mesh
15172 
15173  // Any actions after the adapatation phase
15175 
15176  // Do equation numbering
15177  oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
15178  }
15179 
15180  //========================================================================
15181  /// Refine all submeshes by splitting the elements identified by their
15182  /// pointers within each submesh in a Vector of Vectors, then
15183  /// rebuild the problem.
15184  //========================================================================
15186  const Vector<Vector<RefineableElement*>>& elements_to_be_refined_pt)
15187  {
15189 
15190  // Number of submeshes?
15191  unsigned n_mesh = nsub_mesh();
15192 
15193  // Refine all submeshes if possible
15194  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
15195  {
15196  if (TreeBasedRefineableMeshBase* mmesh_pt =
15197  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(i_mesh)))
15198  {
15199  mmesh_pt->refine_selected_elements(elements_to_be_refined_pt[i_mesh]);
15200  }
15201  else
15202  {
15203  oomph_info << "Info/Warning: Mesh cannot be refined " << std::endl;
15204  }
15205  }
15206 
15207  // Rebuild the global mesh
15209 
15210  // Any actions after the adapatation phase
15212 
15213  // Do equation numbering
15214  oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
15215  }
15216 
15217  //========================================================================
15218  /// p-refine (one and only!) mesh by refining the elements identified
15219  /// by their numbers relative to the problems' only mesh, then rebuild
15220  /// the problem.
15221  //========================================================================
15223  const Vector<unsigned>& elements_to_be_refined)
15224  {
15226 
15227  // Number of submeshes?
15228  unsigned Nmesh = nsub_mesh();
15229 
15230  // Single mesh:
15231  if (Nmesh == 0)
15232  {
15233  // Refine single mesh if possible
15234  if (TreeBasedRefineableMeshBase* mmesh_pt =
15235  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(0)))
15236  {
15237  mmesh_pt->p_refine_selected_elements(elements_to_be_refined);
15238  }
15239  else
15240  {
15241  oomph_info << "Info/Warning: Mesh cannot be refined " << std::endl;
15242  }
15243  }
15244  // Multiple submeshes
15245  else
15246  {
15247  std::ostringstream error_message;
15248  error_message
15249  << "Problem::p_refine_selected_elements(...) only works for\n"
15250  << "multiple-mesh problems if you specify the mesh\n"
15251  << "number in the function argument before the Vector,\n"
15252  << "or a Vector of Vectors for each submesh.\n"
15253  << std::endl;
15254  throw OomphLibError(
15255  error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
15256  }
15257 
15258  // Any actions after the adapatation phase
15260 
15261  // Attach the boundary conditions to the mesh
15262  oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
15263  }
15264 
15265  //========================================================================
15266  /// p-refine (one and only!) mesh by refining the elements identified
15267  /// by their pointers, then rebuild the problem.
15268  //========================================================================
15270  const Vector<PRefineableElement*>& elements_to_be_refined_pt)
15271  {
15273 
15274  // Number of submeshes?
15275  unsigned Nmesh = nsub_mesh();
15276 
15277  // Single mesh:
15278  if (Nmesh == 0)
15279  {
15280  // Refine single mesh if possible
15281  if (TreeBasedRefineableMeshBase* mmesh_pt =
15282  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(0)))
15283  {
15284  mmesh_pt->p_refine_selected_elements(elements_to_be_refined_pt);
15285  }
15286  else
15287  {
15288  oomph_info << "Info/Warning: Mesh cannot be refined " << std::endl;
15289  }
15290  }
15291  // Multiple submeshes
15292  else
15293  {
15294  std::ostringstream error_message;
15295  error_message
15296  << "Problem::p_refine_selected_elements(...) only works for\n"
15297  << "multiple-mesh problems if you specify the mesh\n"
15298  << "number in the function argument before the Vector,\n"
15299  << "or a Vector of Vectors for each submesh.\n"
15300  << std::endl;
15301  throw OomphLibError(
15302  error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
15303  }
15304 
15305  // Any actions after the adapatation phase
15307 
15308  // Do equation numbering
15309  oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
15310  }
15311 
15312  //========================================================================
15313  /// p-refine specified submesh by refining the elements identified
15314  /// by their numbers relative to the specified mesh, then rebuild the problem.
15315  //========================================================================
15317  const unsigned& i_mesh, const Vector<unsigned>& elements_to_be_refined)
15318  {
15320  "p-refinement for multiple submeshes has not yet been tested.",
15321  "Problem::p_refine_selected_elements()",
15322  OOMPH_EXCEPTION_LOCATION);
15323 
15325 
15326  // Number of submeshes?
15327  unsigned n_mesh = nsub_mesh();
15328 
15329  if (i_mesh >= n_mesh)
15330  {
15331  std::ostringstream error_message;
15332  error_message << "Problem only has " << n_mesh
15333  << " submeshes. Cannot p-refine submesh " << i_mesh
15334  << std::endl;
15335  throw OomphLibError(
15336  error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
15337  }
15338 
15339  // Refine single mesh if possible
15340  if (TreeBasedRefineableMeshBase* mmesh_pt =
15341  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(i_mesh)))
15342  {
15343  mmesh_pt->p_refine_selected_elements(elements_to_be_refined);
15344  }
15345  else
15346  {
15347  oomph_info << "Info/Warning: Mesh cannot be refined " << std::endl;
15348  }
15349 
15350  if (n_mesh > 1)
15351  {
15352  // Rebuild the global mesh
15354  }
15355 
15356  // Any actions after the adapatation phase
15358 
15359  // Do equation numbering
15360  oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
15361  }
15362 
15363 
15364  //========================================================================
15365  /// p-refine specified submesh by refining the elements identified
15366  /// by their pointers, then rebuild the problem.
15367  //========================================================================
15369  const unsigned& i_mesh,
15370  const Vector<PRefineableElement*>& elements_to_be_refined_pt)
15371  {
15373  "p-refinement for multiple submeshes has not yet been tested.",
15374  "Problem::p_refine_selected_elements()",
15375  OOMPH_EXCEPTION_LOCATION);
15376 
15378 
15379  // Number of submeshes?
15380  unsigned n_mesh = nsub_mesh();
15381 
15382  if (i_mesh >= n_mesh)
15383  {
15384  std::ostringstream error_message;
15385  error_message << "Problem only has " << n_mesh
15386  << " submeshes. Cannot p-refine submesh " << i_mesh
15387  << std::endl;
15388  throw OomphLibError(
15389  error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
15390  }
15391 
15392  // Refine single mesh if possible
15393  if (TreeBasedRefineableMeshBase* mmesh_pt =
15394  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(i_mesh)))
15395  {
15396  mmesh_pt->p_refine_selected_elements(elements_to_be_refined_pt);
15397  }
15398  else
15399  {
15400  oomph_info << "Info/Warning: Mesh cannot be refined " << std::endl;
15401  }
15402 
15403  if (n_mesh > 1)
15404  {
15405  // Rebuild the global mesh
15407  }
15408 
15409  // Any actions after the adapatation phase
15411 
15412  // Do equation numbering
15413  oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
15414  }
15415 
15416  //========================================================================
15417  /// p-refine all submeshes by refining the elements identified by their
15418  /// numbers relative to each submesh in a Vector of Vectors, then
15419  /// rebuild the problem.
15420  //========================================================================
15422  const Vector<Vector<unsigned>>& elements_to_be_refined)
15423  {
15425  "p-refinement for multiple submeshes has not yet been tested.",
15426  "Problem::p_refine_selected_elements()",
15427  OOMPH_EXCEPTION_LOCATION);
15428 
15430 
15431  // Number of submeshes?
15432  unsigned n_mesh = nsub_mesh();
15433 
15434  // Refine all submeshes if possible
15435  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
15436  {
15437  if (TreeBasedRefineableMeshBase* mmesh_pt =
15438  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(i_mesh)))
15439  {
15440  mmesh_pt->p_refine_selected_elements(elements_to_be_refined[i_mesh]);
15441  }
15442  else
15443  {
15444  oomph_info << "Info/Warning: Mesh cannot be refined " << std::endl;
15445  }
15446  }
15447 
15448  // Rebuild the global mesh
15450 
15451  // Any actions after the adapatation phase
15453 
15454  // Do equation numbering
15455  oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
15456  }
15457 
15458  //========================================================================
15459  /// p-refine all submeshes by refining the elements identified by their
15460  /// pointers within each submesh in a Vector of Vectors, then
15461  /// rebuild the problem.
15462  //========================================================================
15464  const Vector<Vector<PRefineableElement*>>& elements_to_be_refined_pt)
15465  {
15467  "p-refinement for multiple submeshes has not yet been tested.",
15468  "Problem::p_refine_selected_elements()",
15469  OOMPH_EXCEPTION_LOCATION);
15470 
15472 
15473  // Number of submeshes?
15474  unsigned n_mesh = nsub_mesh();
15475 
15476  // Refine all submeshes if possible
15477  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
15478  {
15479  if (TreeBasedRefineableMeshBase* mmesh_pt =
15480  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(i_mesh)))
15481  {
15482  mmesh_pt->p_refine_selected_elements(elements_to_be_refined_pt[i_mesh]);
15483  }
15484  else
15485  {
15486  oomph_info << "Info/Warning: Mesh cannot be refined " << std::endl;
15487  }
15488  }
15489 
15490  // Rebuild the global mesh
15492 
15493  // Any actions after the adapatation phase
15495 
15496  // Do equation numbering
15497  oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
15498  }
15499 
15500 
15501  //========================================================================
15502  /// Helper function to do compund refinement of (all) refineable
15503  /// (sub)mesh(es) uniformly as many times as specified in vector and
15504  /// rebuild problem; doc refinement process. Set boolean argument
15505  /// to true if you want to prune immediately after refining the meshes
15506  /// individually.
15507  //========================================================================
15508  void Problem::refine_uniformly_aux(const Vector<unsigned>& nrefine_for_mesh,
15509  DocInfo& doc_info,
15510  const bool& prune)
15511  {
15512  double t_start = 0.0;
15514  {
15515  t_start = TimingHelpers::timer();
15516  }
15517 
15519 
15520  double t_end = 0.0;
15522  {
15523  t_end = TimingHelpers::timer();
15524  oomph_info
15525  << "Time for actions before adapt in Problem::refine_uniformly_aux(): "
15526  << t_end - t_start << std::endl;
15527  t_start = TimingHelpers::timer();
15528  }
15529 
15530  // Number of submeshes?
15531  unsigned n_mesh = nsub_mesh();
15532 
15533  // Single mesh:
15534  if (n_mesh == 0)
15535  {
15536  // Refine single mesh uniformly if possible
15537  if (RefineableMeshBase* mmesh_pt =
15538  dynamic_cast<RefineableMeshBase*>(mesh_pt(0)))
15539  {
15540  unsigned nref = nrefine_for_mesh[0];
15541  for (unsigned i = 0; i < nref; i++)
15542  {
15543  mmesh_pt->refine_uniformly(doc_info);
15544  }
15545  }
15546  else
15547  {
15548  oomph_info << "Info/Warning: Mesh cannot be refined uniformly "
15549  << std::endl;
15550  }
15551  }
15552  // Multiple submeshes
15553  else
15554  {
15555  // Loop over submeshes
15556  for (unsigned imesh = 0; imesh < n_mesh; imesh++)
15557  {
15558  // Refine i-th submesh uniformly if possible
15559  if (RefineableMeshBase* mmesh_pt =
15560  dynamic_cast<RefineableMeshBase*>(mesh_pt(imesh)))
15561  {
15562  unsigned nref = nrefine_for_mesh[imesh];
15563  for (unsigned i = 0; i < nref; i++)
15564  {
15565  mmesh_pt->refine_uniformly(doc_info);
15566  }
15567  }
15568  else
15569  {
15570  oomph_info << "Info/Warning: Cannot refine mesh " << imesh
15571  << std::endl;
15572  }
15573  }
15574  // Rebuild the global mesh
15576  }
15577 
15579  {
15580  t_end = TimingHelpers::timer();
15581  oomph_info << "Time for mesh-level mesh refinement in "
15582  << "Problem::refine_uniformly_aux(): " << t_end - t_start
15583  << std::endl;
15584  t_start = TimingHelpers::timer();
15585  }
15586 
15587  // Any actions after the adaptation phase
15589 
15590 
15592  {
15593  t_end = TimingHelpers::timer();
15594  oomph_info
15595  << "Time for actions after adapt Problem::refine_uniformly_aux(): "
15596  << t_end - t_start << std::endl;
15597  t_start = TimingHelpers::timer();
15598  }
15599 
15600 
15601 #ifdef OOMPH_HAS_MPI
15602 
15603  // Prune it?
15604  if (prune)
15605  {
15606  // Note: This calls assign eqn numbers already...
15610 
15612  {
15613  t_end = TimingHelpers::timer();
15614  oomph_info << "Time for Problem::prune_halo_elements_and_nodes() in "
15615  << "Problem::refine_uniformly_aux(): " << t_end - t_start
15616  << std::endl;
15617  }
15618  }
15619  else
15620 #else
15621  if (prune)
15622  {
15623  std::ostringstream error_message;
15624  error_message
15625  << "Requested pruning in serial build. Ignoring the request.\n";
15626  OomphLibWarning(error_message.str(),
15627  "Problem::refine_uniformly_aux()",
15628  OOMPH_EXCEPTION_LOCATION);
15629  }
15630 #endif
15631  {
15632  // Do equation numbering
15633  oomph_info
15634  << "Number of equations after Problem::refine_uniformly_aux(): "
15635  << assign_eqn_numbers() << std::endl;
15636 
15638  {
15639  t_end = TimingHelpers::timer();
15640  oomph_info << "Time for Problem::assign_eqn_numbers() in "
15641  << "Problem::refine_uniformly_aux(): " << t_end - t_start
15642  << std::endl;
15643  }
15644  }
15645  }
15646 
15647 
15648  //========================================================================
15649  /// Helper function to do compund p-refinement of (all) p-refineable
15650  /// (sub)mesh(es) uniformly as many times as specified in vector and
15651  /// rebuild problem; doc refinement process. Set boolean argument
15652  /// to true if you want to prune immediately after refining the meshes
15653  /// individually.
15654  //========================================================================
15656  DocInfo& doc_info,
15657  const bool& prune)
15658  {
15659  double t_start = 0.0;
15661  {
15662  t_start = TimingHelpers::timer();
15663  }
15664 
15666 
15667  double t_end = 0.0;
15669  {
15670  t_end = TimingHelpers::timer();
15671  oomph_info << "Time for actions before adapt in "
15672  "Problem::p_refine_uniformly_aux(): "
15673  << t_end - t_start << std::endl;
15674  t_start = TimingHelpers::timer();
15675  }
15676 
15677  // Number of submeshes?
15678  unsigned n_mesh = nsub_mesh();
15679 
15680  // Single mesh:
15681  if (n_mesh == 0)
15682  {
15683  // Refine single mesh uniformly if possible
15684  if (RefineableMeshBase* mmesh_pt =
15685  dynamic_cast<RefineableMeshBase*>(mesh_pt(0)))
15686  {
15687  unsigned nref = nrefine_for_mesh[0];
15688  for (unsigned i = 0; i < nref; i++)
15689  {
15690  mmesh_pt->p_refine_uniformly(doc_info);
15691  }
15692  }
15693  else
15694  {
15695  oomph_info << "Info/Warning: Mesh cannot be p-refined uniformly "
15696  << std::endl;
15697  }
15698  }
15699  // Multiple submeshes
15700  else
15701  {
15703  "p-refinement for multiple submeshes has not yet been tested.",
15704  "Problem::p_refine_uniformly_aux()",
15705  OOMPH_EXCEPTION_LOCATION);
15706 
15707  // Loop over submeshes
15708  for (unsigned imesh = 0; imesh < n_mesh; imesh++)
15709  {
15710  // Refine i-th submesh uniformly if possible
15711  if (RefineableMeshBase* mmesh_pt =
15712  dynamic_cast<RefineableMeshBase*>(mesh_pt(imesh)))
15713  {
15714  unsigned nref = nrefine_for_mesh[imesh];
15715  for (unsigned i = 0; i < nref; i++)
15716  {
15717  mmesh_pt->p_refine_uniformly(doc_info);
15718  }
15719  }
15720  else
15721  {
15722  oomph_info << "Info/Warning: Cannot p-refine mesh " << imesh
15723  << std::endl;
15724  }
15725  }
15726  // Rebuild the global mesh
15728  }
15729 
15731  {
15732  t_end = TimingHelpers::timer();
15733  oomph_info << "Time for mesh-level mesh refinement in "
15734  << "Problem::p_refine_uniformly_aux(): " << t_end - t_start
15735  << std::endl;
15736  t_start = TimingHelpers::timer();
15737  }
15738 
15739  // Any actions after the adaptation phase
15741 
15742 
15744  {
15745  t_end = TimingHelpers::timer();
15746  oomph_info
15747  << "Time for actions after adapt Problem::p_refine_uniformly_aux(): "
15748  << t_end - t_start << std::endl;
15749  t_start = TimingHelpers::timer();
15750  }
15751 
15752 
15753 #ifdef OOMPH_HAS_MPI
15754 
15755  // Prune it?
15756  if (prune)
15757  {
15758  // Note: This calls assign eqn numbers already...
15762 
15764  {
15765  t_end = TimingHelpers::timer();
15766  oomph_info << "Time for Problem::prune_halo_elements_and_nodes() in "
15767  << "Problem::p_refine_uniformly_aux(): " << t_end - t_start
15768  << std::endl;
15769  }
15770  }
15771  else
15772 #else
15773  if (prune)
15774  {
15775  std::ostringstream error_message;
15776  error_message
15777  << "Requested pruning in serial build. Ignoring the request.\n";
15778  OomphLibWarning(error_message.str(),
15779  "Problem::p_refine_uniformly_aux()",
15780  OOMPH_EXCEPTION_LOCATION);
15781  }
15782 #endif
15783  {
15784  // Do equation numbering
15785  oomph_info
15786  << "Number of equations after Problem::p_refine_uniformly_aux(): "
15787  << assign_eqn_numbers() << std::endl;
15788 
15790  {
15791  t_end = TimingHelpers::timer();
15792  oomph_info << "Time for Problem::assign_eqn_numbers() in "
15793  << "Problem::p_refine_uniformly_aux(): " << t_end - t_start
15794  << std::endl;
15795  }
15796  }
15797  }
15798 
15799  //========================================================================
15800  /// Refine submesh i_mesh uniformly and rebuild problem;
15801  /// doc refinement process.
15802  //========================================================================
15803  void Problem::refine_uniformly(const unsigned& i_mesh, DocInfo& doc_info)
15804  {
15806 
15807 #ifdef PARANOID
15808  // Number of submeshes?
15809  if (i_mesh >= nsub_mesh())
15810  {
15811  std::ostringstream error_message;
15812  error_message << "imesh " << i_mesh
15813  << " is greater than the number of sub meshes "
15814  << nsub_mesh() << std::endl;
15815 
15816  throw OomphLibError(
15817  error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
15818  }
15819 #endif
15820 
15821  // Refine single mesh uniformly if possible
15822  if (RefineableMeshBase* mmesh_pt =
15823  dynamic_cast<RefineableMeshBase*>(mesh_pt(i_mesh)))
15824  {
15825  mmesh_pt->refine_uniformly(doc_info);
15826  }
15827  else
15828  {
15829  oomph_info << "Info/Warning: Mesh cannot be refined uniformly "
15830  << std::endl;
15831  }
15832 
15833  // Rebuild the global mesh
15835 
15836  // Any actions after the adaptation phase
15838 
15839  // Do equation numbering
15840  oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
15841  }
15842 
15843  //========================================================================
15844  /// p-refine submesh i_mesh uniformly and rebuild problem;
15845  /// doc refinement process.
15846  //========================================================================
15847  void Problem::p_refine_uniformly(const unsigned& i_mesh, DocInfo& doc_info)
15848  {
15850 
15851 #ifdef PARANOID
15852  // Number of submeshes?
15853  if (i_mesh >= nsub_mesh())
15854  {
15855  std::ostringstream error_message;
15856  error_message << "imesh " << i_mesh
15857  << " is greater than the number of sub meshes "
15858  << nsub_mesh() << std::endl;
15859 
15860  throw OomphLibError(
15861  error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
15862  }
15863 #endif
15864 
15865  // Refine single mesh uniformly if possible
15866  if (RefineableMeshBase* mmesh_pt =
15867  dynamic_cast<RefineableMeshBase*>(mesh_pt(i_mesh)))
15868  {
15869  mmesh_pt->p_refine_uniformly(doc_info);
15870  }
15871  else
15872  {
15873  oomph_info << "Info/Warning: Mesh cannot be refined uniformly "
15874  << std::endl;
15875  }
15876 
15877  // Rebuild the global mesh
15879 
15880  // Any actions after the adaptation phase
15882 
15883  // Do equation numbering
15884  oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
15885  }
15886 
15887 
15888  //========================================================================
15889  /// Unrefine (all) refineable (sub)mesh(es) uniformly and rebuild problem.
15890  /// Return 0 for success,
15891  /// 1 for failure (if unrefinement has reached the coarsest permitted
15892  /// level)
15893  //========================================================================
15895  {
15896  // Call actions_before_adapt()
15898 
15899  // Has unrefinement been successful?
15900  unsigned success_flag = 0;
15901 
15902  // Number of submeshes?
15903  unsigned n_mesh = nsub_mesh();
15904 
15905  // Single mesh:
15906  if (n_mesh == 0)
15907  {
15908  // Unrefine single mesh uniformly if possible
15909  if (RefineableMeshBase* mmesh_pt =
15910  dynamic_cast<RefineableMeshBase*>(mesh_pt(0)))
15911  {
15912  success_flag += mmesh_pt->unrefine_uniformly();
15913  }
15914  else
15915  {
15916  oomph_info << "Info/Warning: Mesh cannot be unrefined uniformly "
15917  << std::endl;
15918  }
15919  }
15920  // Multiple submeshes
15921  else
15922  {
15923  // Loop over submeshes
15924  for (unsigned imesh = 0; imesh < n_mesh; imesh++)
15925  {
15926  // Unrefine i-th submesh uniformly if possible
15927  if (RefineableMeshBase* mmesh_pt =
15928  dynamic_cast<RefineableMeshBase*>(mesh_pt(imesh)))
15929  {
15930  success_flag += mmesh_pt->unrefine_uniformly();
15931  }
15932  else
15933  {
15934  oomph_info << "Info/Warning: Cannot unrefine mesh " << imesh
15935  << std::endl;
15936  }
15937  }
15938  // Rebuild the global mesh
15940  }
15941 
15942  // Any actions after the adaptation phase
15944 
15945  // Do equation numbering
15946  oomph_info << " Number of equations: " << assign_eqn_numbers() << std::endl;
15947 
15948  // Judge success
15949  if (success_flag > 0)
15950  {
15951  return 1;
15952  }
15953  else
15954  {
15955  return 0;
15956  }
15957  }
15958 
15959  //========================================================================
15960  /// Unrefine submesh i_mesh uniformly and rebuild problem.
15961  /// Return 0 for success,
15962  /// 1 for failure (if unrefinement has reached the coarsest permitted
15963  /// level)
15964  //========================================================================
15965  unsigned Problem::unrefine_uniformly(const unsigned& i_mesh)
15966  {
15968 
15969  // Has unrefinement been successful?
15970  unsigned success_flag = 0;
15971 
15972 #ifdef PARANOID
15973  // Number of submeshes?
15974  if (i_mesh >= nsub_mesh())
15975  {
15976  std::ostringstream error_message;
15977  error_message << "imesh " << i_mesh
15978  << " is greater than the number of sub meshes "
15979  << nsub_mesh() << std::endl;
15980 
15981  throw OomphLibError(
15982  error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
15983  }
15984 #endif
15985 
15986  // Unrefine single mesh uniformly if possible
15987  if (RefineableMeshBase* mmesh_pt =
15988  dynamic_cast<RefineableMeshBase*>(mesh_pt(i_mesh)))
15989  {
15990  success_flag += mmesh_pt->unrefine_uniformly();
15991  }
15992  else
15993  {
15994  oomph_info << "Info/Warning: Mesh cannot be unrefined uniformly "
15995  << std::endl;
15996  }
15997 
15998  // Rebuild the global mesh
16000 
16001  // Any actions after the adaptation phase
16003 
16004  // Do equation numbering
16005  oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
16006 
16007  // Judge success
16008  if (success_flag > 0)
16009  {
16010  return 1;
16011  }
16012  else
16013  {
16014  return 0;
16015  }
16016  }
16017 
16018 
16019  //========================================================================
16020  /// p-unrefine (all) p-refineable (sub)mesh(es) uniformly and rebuild problem;
16021  /// doc refinement process.
16022  //========================================================================
16024  {
16026 
16027  // Number of submeshes?
16028  unsigned n_mesh = nsub_mesh();
16029 
16030  // Single mesh:
16031  if (n_mesh == 0)
16032  {
16033  // Unrefine single mesh uniformly if possible
16034  if (RefineableMeshBase* mmesh_pt =
16035  dynamic_cast<RefineableMeshBase*>(mesh_pt(0)))
16036  {
16037  mmesh_pt->p_unrefine_uniformly(doc_info);
16038  }
16039  else
16040  {
16041  oomph_info << "Info/Warning: Mesh cannot be p-unrefined uniformly "
16042  << std::endl;
16043  }
16044  }
16045  // Multiple submeshes
16046  else
16047  {
16048  // Not tested:
16049  throw OomphLibError("This functionality has not yet been tested.",
16050  OOMPH_CURRENT_FUNCTION,
16051  OOMPH_EXCEPTION_LOCATION);
16052  // Loop over submeshes
16053  for (unsigned imesh = 0; imesh < n_mesh; imesh++)
16054  {
16055  // Unrefine i-th submesh uniformly if possible
16056  if (RefineableMeshBase* mmesh_pt =
16057  dynamic_cast<RefineableMeshBase*>(mesh_pt(imesh)))
16058  {
16059  mmesh_pt->p_unrefine_uniformly(doc_info);
16060  }
16061  else
16062  {
16063  oomph_info << "Info/Warning: Cannot p-unrefine mesh " << imesh
16064  << std::endl;
16065  }
16066  }
16067  // Rebuild the global mesh
16069  }
16070 
16071  // Any actions after the adaptation phase
16073 
16074  // Do equation numbering
16075  oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
16076  }
16077 
16078  //========================================================================
16079  /// p-unrefine submesh i_mesh uniformly and rebuild problem;
16080  /// doc refinement process.
16081  //========================================================================
16082  void Problem::p_unrefine_uniformly(const unsigned& i_mesh, DocInfo& doc_info)
16083  {
16085 
16086 #ifdef PARANOID
16087  // Number of submeshes?
16088  if (i_mesh >= nsub_mesh())
16089  {
16090  std::ostringstream error_message;
16091  error_message << "imesh " << i_mesh
16092  << " is greater than the number of sub meshes "
16093  << nsub_mesh() << std::endl;
16094 
16095  throw OomphLibError(
16096  error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
16097  }
16098 #endif
16099 
16100  // Refine single mesh uniformly if possible
16101  if (RefineableMeshBase* mmesh_pt =
16102  dynamic_cast<RefineableMeshBase*>(mesh_pt(i_mesh)))
16103  {
16104  mmesh_pt->p_unrefine_uniformly(doc_info);
16105  }
16106  else
16107  {
16108  oomph_info << "Info/Warning: Mesh cannot be p-unrefined uniformly "
16109  << std::endl;
16110  }
16111 
16112  // Rebuild the global mesh
16114 
16115  // Any actions after the adaptation phase
16117 
16118  // Do equation numbering
16119  oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
16120  }
16121 
16122 
16123  //========================================================================
16124  /// Do one timestep, dt, forward using Newton's method with specified
16125  /// tolerance and linear solver specified via member data.
16126  /// Keep adapting on all meshes to criteria specified in
16127  /// these meshes (up to max_adapt adaptations are performed).
16128  /// If first_timestep==true, re-set initial conditions after mesh adaptation.
16129  /// Shifting of time can be suppressed by overwriting the
16130  /// default value of shift (true). [Shifting must be done
16131  /// if first_timestep==true because we're constantly re-assigning
16132  /// the initial conditions; if first_timestep==true and shift==false
16133  /// shifting is performed anyway and a warning is issued.
16134  //========================================================================
16135  void Problem::unsteady_newton_solve(const double& dt,
16136  const unsigned& max_adapt,
16137  const bool& first_timestep,
16138  const bool& shift)
16139  {
16140  // Do shifting or not?
16141  bool shift_it = shift;
16142 
16143  // Warning:
16144  if (first_timestep && (!shift) && (!Default_set_initial_condition_called))
16145  {
16146  shift_it = true;
16147  oomph_info
16148  << "\n\n===========================================================\n";
16149  oomph_info << " ******** WARNING *********** \n";
16150  oomph_info
16151  << "===========================================================\n";
16152  oomph_info << "Problem::unsteady_newton_solve() called with "
16153  << std::endl;
16154  oomph_info << "first_timestep: " << first_timestep << std::endl;
16155  oomph_info << "shift: " << shift << std::endl;
16156  oomph_info << "This doesn't make sense (shifting does have to be done"
16157  << std::endl;
16158  oomph_info << "since we're constantly re-assigning the initial conditions"
16159  << std::endl;
16160  oomph_info
16161  << "\n===========================================================\n\n";
16162  }
16163 
16164 
16165  // Find the initial time
16166  double initial_time = time_pt()->time();
16167 
16168  // Max number of solves
16169  unsigned max_solve = max_adapt + 1;
16170 
16171  // Adaptation loop
16172  //----------------
16173  for (unsigned isolve = 0; isolve < max_solve; isolve++)
16174  {
16175  // Only adapt after the first solve has been done!
16176  if (isolve > 0)
16177  {
16178  unsigned n_refined;
16179  unsigned n_unrefined;
16180 
16181  // Adapt problem
16182  adapt(n_refined, n_unrefined);
16183 
16184 #ifdef OOMPH_HAS_MPI
16185  // Adaptation only converges if ALL the processes have no
16186  // refinement or unrefinement to perform
16187  unsigned total_refined = 0;
16188  unsigned total_unrefined = 0;
16190  {
16191  MPI_Allreduce(&n_refined,
16192  &total_refined,
16193  1,
16194  MPI_UNSIGNED,
16195  MPI_SUM,
16196  this->communicator_pt()->mpi_comm());
16197  n_refined = total_refined;
16198  MPI_Allreduce(&n_unrefined,
16199  &total_unrefined,
16200  1,
16201  MPI_UNSIGNED,
16202  MPI_SUM,
16203  this->communicator_pt()->mpi_comm());
16204  n_unrefined = total_unrefined;
16205  }
16206 #endif
16207 
16208  oomph_info << "---> " << n_refined << " elements were refined, and "
16209  << n_unrefined << " were unrefined, in total." << std::endl;
16210 
16211  // Check convergence of adaptation cycle
16212  if ((n_refined == 0) && (n_unrefined == 0))
16213  {
16214  oomph_info << "\n \n Solution is fully converged in "
16215  << "Problem::unsteady_newton_solver() \n \n ";
16216  break;
16217  }
16218 
16219  // Reset the time
16220  time_pt()->time() = initial_time;
16221 
16222  // Reset the inital condition on refined meshes. Note that because we
16223  // have reset the global time to the initial time, the initial
16224  // conditions are reset at time t=0 rather than at time t=dt
16225  if (first_timestep)
16226  {
16227  // Reset default set_initial_condition has been called flag to false
16229 
16230  oomph_info << "Re-setting initial condition " << std::endl;
16232 
16233  // If the default set_initial_condition function has been called,
16234  // we must not shift the timevalues on the first timestep, as we
16235  // will NOT be constantly re-assigning the initial condition
16237  {
16238  shift_it = false;
16239  }
16240  }
16241  }
16242 
16243  // Now do the actual unsteady timestep
16244  // If it's the first time around the loop, or the first timestep
16245  // shift the timevalues, otherwise don't
16246  // Note: we need to shift if it's the first timestep because
16247  // we're constantly re-assigning the initial condition above!
16248  // The only exception to this is if the default set_initial_condition
16249  // function has been called, in which case we must NOT shift!
16250  if ((isolve == 0) || (first_timestep))
16251  {
16252  Problem::unsteady_newton_solve(dt, shift_it);
16253  }
16254  // Subsequent solve: Have shifted already -- don't do it again.
16255  else
16256  {
16257  shift_it = false;
16258  Problem::unsteady_newton_solve(dt, shift_it);
16259  }
16260 
16261  if (isolve == max_solve - 1)
16262  {
16263  oomph_info
16264  << std::endl
16265  << "----------------------------------------------------------"
16266  << std::endl
16267  << "Reached max. number of adaptations in \n"
16268  << "Problem::unsteady_newton_solver().\n"
16269  << "----------------------------------------------------------"
16270  << std::endl
16271  << std::endl;
16272  }
16273 
16274  } // End of adaptation loop
16275  }
16276 
16277 
16278  //========================================================================
16279  /// Adaptive Newton solver.
16280  /// The linear solver takes a pointer to the problem (which defines
16281  /// the Jacobian \b J and the residual Vector \b r) and returns
16282  /// the solution \b x of the system
16283  /// \f[ {\bf J} {\bf x} = - \bf{r} \f].
16284  /// Performs at most max_adapt adaptations on all meshes.
16285  //========================================================================
16286  void Problem::newton_solve(const unsigned& max_adapt)
16287  {
16288  // Max number of solves
16289  unsigned max_solve = max_adapt + 1;
16290 
16291  // Adaptation loop
16292  //----------------
16293  for (unsigned isolve = 0; isolve < max_solve; isolve++)
16294  {
16295  // Only adapt after the first solve has been done!
16296  if (isolve > 0)
16297  {
16298  unsigned n_refined;
16299  unsigned n_unrefined;
16300 
16301  // Adapt problem
16302  adapt(n_refined, n_unrefined);
16303 
16304 #ifdef OOMPH_HAS_MPI
16305  // Adaptation only converges if ALL the processes have no
16306  // refinement or unrefinement to perform
16307  unsigned total_refined = 0;
16308  unsigned total_unrefined = 0;
16310  {
16311  MPI_Allreduce(&n_refined,
16312  &total_refined,
16313  1,
16314  MPI_UNSIGNED,
16315  MPI_SUM,
16316  this->communicator_pt()->mpi_comm());
16317  n_refined = total_refined;
16318  MPI_Allreduce(&n_unrefined,
16319  &total_unrefined,
16320  1,
16321  MPI_UNSIGNED,
16322  MPI_SUM,
16323  this->communicator_pt()->mpi_comm());
16324  n_unrefined = total_unrefined;
16325  }
16326 #endif
16327 
16328  oomph_info << "---> " << n_refined << " elements were refined, and "
16329  << n_unrefined << " were unrefined"
16330 #ifdef OOMPH_HAS_MPI
16331  << ", in total (over all processors).\n";
16332 #else
16333  << ".\n";
16334 #endif
16335 
16336 
16337  // Check convergence of adaptation cycle
16338  if ((n_refined == 0) && (n_unrefined == 0))
16339  {
16340  oomph_info << "\n \n Solution is fully converged in "
16341  << "Problem::newton_solver(). \n \n ";
16342  break;
16343  }
16344  }
16345 
16346 
16347  // Do actual solve
16348  //----------------
16349  {
16350  // Now update anything that needs updating
16351  // NOT NEEDED -- IS CALLED IN newton_solve BELOW! #
16352  // actions_before_newton_solve();
16353 
16354  try
16355  {
16356  // Solve the non-linear problem for this timestep with Newton's method
16357  newton_solve();
16358  }
16359  // Catch any exceptions thrown in the Newton solver
16360  catch (NewtonSolverError& error)
16361  {
16362  oomph_info << std::endl
16363  << "USER-DEFINED ERROR IN NEWTON SOLVER " << std::endl;
16364  // Check to see whether we have reached Max_iterations
16365  if (error.iterations() == Max_newton_iterations)
16366  {
16367  oomph_info << "MAXIMUM NUMBER OF ITERATIONS (" << error.iterations()
16368  << ") REACHED WITHOUT CONVERGENCE " << std::endl;
16369  }
16370  // If not, it must be that we have exceeded the maximum residuals
16371  else
16372  {
16373  oomph_info << "MAXIMUM RESIDUALS: " << error.maxres()
16374  << "EXCEEDS PREDEFINED MAXIMUM " << Max_residuals
16375  << std::endl;
16376  }
16377 
16378  // Die horribly!!
16379  std::ostringstream error_stream;
16380  error_stream << "Error occured in adaptive Newton solver. "
16381  << std::endl;
16382  throw OomphLibError(error_stream.str(),
16383  OOMPH_CURRENT_FUNCTION,
16384  OOMPH_EXCEPTION_LOCATION);
16385  }
16386 
16387  // Now update anything that needs updating
16388  // NOT NEEDED -- WAS CALLED IN newton_solve ABOVE
16389  // !actions_after_newton_solve();
16390 
16391  } // End of solve block
16392 
16393 
16394  if (isolve == max_solve - 1)
16395  {
16396  oomph_info
16397  << std::endl
16398  << "----------------------------------------------------------"
16399  << std::endl
16400  << "Reached max. number of adaptations in \n"
16401  << "Problem::newton_solver().\n"
16402  << "----------------------------------------------------------"
16403  << std::endl
16404  << std::endl;
16405  }
16406 
16407  } // End of adaptation loop
16408  }
16409 
16410  //========================================================================
16411  /// Delete any external storage for any submeshes
16412  /// NB this would ordinarily take place within the adaptation procedure
16413  /// for each submesh (See RefineableMesh::adapt_mesh(...)), but there
16414  /// are instances where the actions_before/after_adapt routines are used
16415  /// and no adaptive routines are called in between (e.g. when doc-ing
16416  /// errors at the end of an adaptive newton solver)
16417  //========================================================================
16419  {
16420  // Number of submeshes
16421  unsigned n_mesh = nsub_mesh();
16422 
16423  // External storage will only exist if there is more than one (sub)mesh
16424  if (n_mesh > 1)
16425  {
16426  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
16427  {
16429  }
16430  }
16431  }
16432 
16433 
16434 #ifdef OOMPH_HAS_MPI
16435 
16436  //====================================================================
16437  /// Get all the halo data stored on this processor and store pointers
16438  /// to the data in a map, indexed by the gobal eqn number
16439  //====================================================================
16440  void Problem::get_all_halo_data(std::map<unsigned, double*>& map_of_halo_data)
16441  {
16442  // Halo data is stored in the meshes, so kick the problem down to that
16443  // level
16444 
16445  // Find the number of meshes
16446  unsigned n_mesh = this->nsub_mesh();
16447  // If there are no submeshes it's only the main mesh
16448  if (n_mesh == 0)
16449  {
16450  mesh_pt()->get_all_halo_data(map_of_halo_data);
16451  }
16452  // Otherwise loop over all the submeshes
16453  else
16454  {
16455  for (unsigned imesh = 0; imesh < n_mesh; ++imesh)
16456  {
16457  mesh_pt(imesh)->get_all_halo_data(map_of_halo_data);
16458  }
16459  }
16460  }
16461 
16462 
16463  //========================================================================
16464  /// Check the halo/haloed/shared node/element schemes.
16465  //========================================================================
16467  {
16468  // The bulk of the stuff that was in this routine is mesh-based, and
16469  // should therefore drop into the Mesh base class. All that needs to remain
16470  // here is a "wrapper" which calls the function dependent upon the number
16471  // of (sub)meshes that may have been distributed.
16472 
16473  unsigned n_mesh = nsub_mesh();
16474 
16475  if (n_mesh == 0)
16476  {
16477  oomph_info << "Checking halo schemes on single mesh" << std::endl;
16478  doc_info.label() = "_one_and_only_mesh_";
16479  mesh_pt()->check_halo_schemes(doc_info,
16481  }
16482  else // there are submeshes
16483  {
16484  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
16485  {
16486  oomph_info << "Checking halo schemes on submesh " << i_mesh
16487  << std::endl;
16488  std::stringstream tmp;
16489  tmp << "_mesh" << i_mesh << "_";
16490  doc_info.label() = tmp.str();
16491  mesh_pt(i_mesh)->check_halo_schemes(doc_info,
16493  }
16494  }
16495  }
16496 
16497 
16498  //========================================================================
16499  /// Synchronise all dofs by calling the appropriate synchronisation
16500  /// routines for all meshes and the assembly handler
16501  //========================================================================
16503  {
16504  // Synchronise dofs themselves
16505  bool do_halos = true;
16506  bool do_external_halos = false;
16507  this->synchronise_dofs(do_halos, do_external_halos);
16508 
16509 
16510  do_halos = false;
16511  do_external_halos = true;
16512  this->synchronise_dofs(do_halos, do_external_halos);
16513 
16514  // Now perform any synchronisation required by the assembly handler
16515  this->assembly_handler_pt()->synchronise();
16516  }
16517 
16518 
16519  //========================================================================
16520  /// Synchronise the degrees of freedom by overwriting
16521  /// the haloed values with their non-halo counterparts held
16522  /// on other processors. Bools control if we deal with data associated with
16523  /// external halo/ed elements/nodes or the "normal" halo/ed ones.
16524  //========================================================================
16525  void Problem::synchronise_dofs(const bool& do_halos,
16526  const bool& do_external_halos)
16527  {
16528  // Do we have submeshes?
16529  unsigned n_mesh_loop = 1;
16530  unsigned nmesh = nsub_mesh();
16531  if (nmesh > 0)
16532  {
16533  n_mesh_loop = nmesh;
16534  }
16535 
16536  // Local storage for number of processors and current processor
16537  const int n_proc = this->communicator_pt()->nproc();
16538 
16539  // If only one processor then return
16540  if (n_proc == 1)
16541  {
16542  return;
16543  }
16544 
16545  const int my_rank = this->communicator_pt()->my_rank();
16546 
16547  // Storage for number of data to be sent to each processor
16548  Vector<int> send_n(n_proc, 0);
16549 
16550  // Storage for all values to be sent to all processors
16551  Vector<double> send_data;
16552 
16553  // Start location within send_data for data to be sent to each processor
16554  Vector<int> send_displacement(n_proc, 0);
16555 
16556  // Loop over all processors
16557  for (int rank = 0; rank < n_proc; rank++)
16558  {
16559  // Set the offset for the current processor
16560  send_displacement[rank] = send_data.size();
16561 
16562  // Don't bother to do anything if the processor in the loop is the
16563  // current processor
16564  if (rank != my_rank)
16565  {
16566  // Deal with sub-meshes one-by-one if required
16567  Mesh* my_mesh_pt = 0;
16568 
16569  // Loop over submeshes
16570  for (unsigned imesh = 0; imesh < n_mesh_loop; imesh++)
16571  {
16572  if (nmesh == 0)
16573  {
16574  my_mesh_pt = mesh_pt();
16575  }
16576  else
16577  {
16578  my_mesh_pt = mesh_pt(imesh);
16579  }
16580 
16581  if (do_halos)
16582  {
16583  // How many of my nodes are haloed by the processor whose values
16584  // are updated?
16585  unsigned n_nod = my_mesh_pt->nhaloed_node(rank);
16586  for (unsigned n = 0; n < n_nod; n++)
16587  {
16588  // Add the data for each haloed node to the vector
16589  my_mesh_pt->haloed_node_pt(rank, n)->add_values_to_vector(
16590  send_data);
16591  }
16592 
16593  // Now loop over haloed elements and prepare to add their
16594  // internal data to the big vector to be sent
16595  Vector<GeneralisedElement*> haloed_elem_pt =
16596  my_mesh_pt->haloed_element_pt(rank);
16597  unsigned nelem_haloed = haloed_elem_pt.size();
16598  for (unsigned e = 0; e < nelem_haloed; e++)
16599  {
16600  haloed_elem_pt[e]->add_internal_data_values_to_vector(send_data);
16601  }
16602  }
16603 
16604  if (do_external_halos)
16605  {
16606  // How many of my nodes are externally haloed by the processor whose
16607  // values are updated? NB these nodes are on the external mesh.
16608  unsigned n_ext_nod = my_mesh_pt->nexternal_haloed_node(rank);
16609  for (unsigned n = 0; n < n_ext_nod; n++)
16610  {
16611  // Add data from each external haloed node to the vector
16612  my_mesh_pt->external_haloed_node_pt(rank, n)
16613  ->add_values_to_vector(send_data);
16614  }
16615 
16616  // Now loop over haloed elements and prepare to send internal data
16617  unsigned next_elem_haloed =
16618  my_mesh_pt->nexternal_haloed_element(rank);
16619  for (unsigned e = 0; e < next_elem_haloed; e++)
16620  {
16621  my_mesh_pt->external_haloed_element_pt(rank, e)
16623  }
16624  }
16625  } // end of loop over meshes
16626  }
16627 
16628  // Find the number of data added to the vector
16629  send_n[rank] = send_data.size() - send_displacement[rank];
16630  }
16631 
16632 
16633  // Storage for the number of data to be received from each processor
16634  Vector<int> receive_n(n_proc, 0);
16635 
16636  // Now send numbers of data to be sent between all processors
16637  MPI_Alltoall(&send_n[0],
16638  1,
16639  MPI_INT,
16640  &receive_n[0],
16641  1,
16642  MPI_INT,
16643  this->communicator_pt()->mpi_comm());
16644 
16645  // We now prepare the data to be received
16646  // by working out the displacements from the received data
16647  Vector<int> receive_displacement(n_proc, 0);
16648  int receive_data_count = 0;
16649  for (int rank = 0; rank < n_proc; ++rank)
16650  {
16651  // Displacement is number of data received so far
16652  receive_displacement[rank] = receive_data_count;
16653  receive_data_count += receive_n[rank];
16654  }
16655 
16656  // Now resize the receive buffer for all data from all processors
16657  // Make sure that it has a size of at least one
16658  if (receive_data_count == 0)
16659  {
16660  ++receive_data_count;
16661  }
16662  Vector<double> receive_data(receive_data_count);
16663 
16664  // Make sure that the send buffer has size at least one
16665  // so that we don't get a segmentation fault
16666  if (send_data.size() == 0)
16667  {
16668  send_data.resize(1);
16669  }
16670 
16671  // Now send the data between all the processors
16672  MPI_Alltoallv(&send_data[0],
16673  &send_n[0],
16674  &send_displacement[0],
16675  MPI_DOUBLE,
16676  &receive_data[0],
16677  &receive_n[0],
16678  &receive_displacement[0],
16679  MPI_DOUBLE,
16680  this->communicator_pt()->mpi_comm());
16681 
16682  // Now use the received data to update the halo nodes
16683  for (int send_rank = 0; send_rank < n_proc; send_rank++)
16684  {
16685  // Don't bother to do anything for the processor corresponding to the
16686  // current processor or if no data were received from this processor
16687  if ((send_rank != my_rank) && (receive_n[send_rank] != 0))
16688  {
16689  // Counter for the data within the large array
16690  unsigned count = receive_displacement[send_rank];
16691 
16692  // Deal with sub-meshes one-by-one if required
16693  Mesh* my_mesh_pt = 0;
16694 
16695  // Loop over submeshes
16696  for (unsigned imesh = 0; imesh < n_mesh_loop; imesh++)
16697  {
16698  if (nmesh == 0)
16699  {
16700  my_mesh_pt = mesh_pt();
16701  }
16702  else
16703  {
16704  my_mesh_pt = mesh_pt(imesh);
16705  }
16706 
16707  if (do_halos)
16708  {
16709  // How many of my nodes are halos whose non-halo counter
16710  // parts live on processor send_rank?
16711  unsigned n_nod = my_mesh_pt->nhalo_node(send_rank);
16712  for (unsigned n = 0; n < n_nod; n++)
16713  {
16714  // Read in values for each halo node
16715  my_mesh_pt->halo_node_pt(send_rank, n)
16716  ->read_values_from_vector(receive_data, count);
16717  }
16718 
16719  // Get number of halo elements whose non-halo is
16720  // on process send_rank
16721  Vector<GeneralisedElement*> halo_elem_pt =
16722  my_mesh_pt->halo_element_pt(send_rank);
16723 
16724  unsigned nelem_halo = halo_elem_pt.size();
16725  for (unsigned e = 0; e < nelem_halo; e++)
16726  {
16727  halo_elem_pt[e]->read_internal_data_values_from_vector(
16728  receive_data, count);
16729  }
16730  }
16731 
16732  if (do_external_halos)
16733  {
16734  // How many of my nodes are external halos whose external non-halo
16735  // counterparts live on processor send_rank?
16736  unsigned n_ext_nod = my_mesh_pt->nexternal_halo_node(send_rank);
16737 
16738  // Copy into the values of the external halo nodes
16739  // on the present processors
16740  for (unsigned n = 0; n < n_ext_nod; n++)
16741  {
16742  // Read the data from the array into each halo node
16743  my_mesh_pt->external_halo_node_pt(send_rank, n)
16744  ->read_values_from_vector(receive_data, count);
16745  }
16746 
16747  // Get number of halo elements whose non-halo is
16748  // on process send_rank
16749  unsigned next_elem_halo =
16750  my_mesh_pt->nexternal_halo_element(send_rank);
16751  for (unsigned e = 0; e < next_elem_halo; e++)
16752  {
16753  my_mesh_pt->external_halo_element_pt(send_rank, e)
16754  ->read_internal_data_values_from_vector(receive_data, count);
16755  }
16756  }
16757 
16758  } // end of loop over meshes
16759  }
16760  } // End of data is received
16761  } // End of synchronise
16762 
16763 
16764  //========================================================================
16765  /// Synchronise equation numbers and return the total
16766  /// number of degrees of freedom in the overall problem
16767  //========================================================================
16768  long Problem::synchronise_eqn_numbers(const bool& assign_local_eqn_numbers)
16769  {
16770  // number of equations on this processor, which at this stage is only known
16771  // by counting the number of dofs that have been added to the problem
16772  unsigned my_n_eqn = Dof_pt.size();
16773 
16774  // my rank
16775  unsigned my_rank = Communicator_pt->my_rank();
16776 
16777  // number of processors
16778  unsigned nproc = Communicator_pt->nproc();
16779 
16780  // // Time alternative communication
16781  // Vector<unsigned> n_eqn(nproc);
16782  // {
16783  // double t_start = TimingHelpers::timer();
16784 
16785  // // Gather numbers of equations (enumerated independently on all procs)
16786  // MPI_Allgather(&my_n_eqn,1,MPI_UNSIGNED,&n_eqn[0],
16787  // 1,MPI_INT,Communicator_pt->mpi_comm());
16788 
16789  // double t_end = TimingHelpers::timer();
16790  // oomph_info << "Time for allgather-based exchange of eqn numbers: "
16791  // << t_end-t_start << std::endl;
16792  // }
16793 
16794  double t_start = TimingHelpers::timer();
16795 
16796  // send my_n_eqn to with rank greater than my_rank
16797  unsigned n_send = nproc - my_rank - 1;
16798  Vector<MPI_Request> send_req(n_send);
16799  for (unsigned p = my_rank + 1; p < nproc; p++)
16800  {
16801  MPI_Isend(&my_n_eqn,
16802  1,
16803  MPI_UNSIGNED,
16804  p,
16805  0,
16806  Communicator_pt->mpi_comm(),
16807  &send_req[p - my_rank - 1]);
16808  }
16809 
16810  // recv n_eqn from processors with rank less than my_rank
16811  Vector<unsigned> n_eqn_on_proc(my_rank);
16812  for (unsigned p = 0; p < my_rank; p++)
16813  {
16814  MPI_Recv(&n_eqn_on_proc[p],
16815  1,
16816  MPI_UNSIGNED,
16817  p,
16818  0,
16819  Communicator_pt->mpi_comm(),
16820  MPI_STATUS_IGNORE);
16821  }
16822 
16823  double t_end = 0.0;
16825  {
16826  t_end = TimingHelpers::timer();
16827  oomph_info << "Time for send and receive stuff: " << t_end - t_start
16828  << std::endl;
16829  t_start = TimingHelpers::timer();
16830  }
16831 
16832  // determine the number of equation on processors with rank
16833  // less than my_rank
16834  unsigned my_eqn_num_base = 0;
16835  for (unsigned p = 0; p < my_rank; p++)
16836  {
16837  my_eqn_num_base += n_eqn_on_proc[p];
16838  // if (n_eqn_on_proc[p]!=n_eqn[p])
16839  // {
16840  // std::cout << "proc " << my_rank << "clash in eqn numbers: "
16841  // << p << " " << n_eqn_on_proc[p] << " " << n_eqn[p]
16842  // << std::endl;
16843  // }
16844  }
16845 
16846  // Loop over all internal data (on elements) and bump up their
16847  // equation numbers if they exist
16848  unsigned nelem = mesh_pt()->nelement();
16849  for (unsigned e = 0; e < nelem; e++)
16850  {
16851  GeneralisedElement* el_pt = mesh_pt()->element_pt(e);
16852 
16853  unsigned nintern_data = el_pt->ninternal_data();
16854  for (unsigned iintern = 0; iintern < nintern_data; iintern++)
16855  {
16856  Data* int_data_pt = el_pt->internal_data_pt(iintern);
16857  unsigned nval = int_data_pt->nvalue();
16858  for (unsigned ival = 0; ival < nval; ival++)
16859  {
16860  int old_eqn_number = int_data_pt->eqn_number(ival);
16861  if (old_eqn_number >= 0) // i.e. it's being used
16862  {
16863  // Bump up eqn number
16864  int new_eqn_number = old_eqn_number + my_eqn_num_base;
16865  int_data_pt->eqn_number(ival) = new_eqn_number;
16866  }
16867  }
16868  }
16869  }
16870 
16871  // Loop over all nodes on current processor and bump up their
16872  // equation numbers if they're not pinned!
16873  unsigned nnod = mesh_pt()->nnode();
16874  for (unsigned j = 0; j < nnod; j++)
16875  {
16876  Node* nod_pt = mesh_pt()->node_pt(j);
16877 
16878  // loop over ALL eqn numbers - variable number of values
16879  unsigned nval = nod_pt->nvalue();
16880 
16881  for (unsigned ival = 0; ival < nval; ival++)
16882  {
16883  int old_eqn_number = nod_pt->eqn_number(ival);
16884  // Include all eqn numbers
16885  if (old_eqn_number >= 0)
16886  {
16887  // Bump up eqn number
16888  int new_eqn_number = old_eqn_number + my_eqn_num_base;
16889  nod_pt->eqn_number(ival) = new_eqn_number;
16890  }
16891  }
16892 
16893  // Is this a solid node? If so, need to bump up its equation number(s)
16894  SolidNode* solid_nod_pt = dynamic_cast<SolidNode*>(nod_pt);
16895 
16896  if (solid_nod_pt != 0)
16897  {
16898  // Find equation numbers
16899  unsigned nval = solid_nod_pt->variable_position_pt()->nvalue();
16900  for (unsigned ival = 0; ival < nval; ival++)
16901  {
16902  int old_eqn_number =
16903  solid_nod_pt->variable_position_pt()->eqn_number(ival);
16904  // include all eqn numbers
16905 
16906  if (old_eqn_number >= 0)
16907  {
16908  // Bump up eqn number
16909  int new_eqn_number = old_eqn_number + my_eqn_num_base;
16910  solid_nod_pt->variable_position_pt()->eqn_number(ival) =
16911  new_eqn_number;
16912  }
16913  }
16914  }
16915  }
16916 
16918  {
16919  t_end = TimingHelpers::timer();
16920  oomph_info << "Time for bumping: " << t_end - t_start << std::endl;
16921  t_start = TimingHelpers::timer();
16922  }
16923 
16924 
16925  // Now copy the haloed eqn numbers across
16926  // This has to include the internal data equation numbers as well
16927  // as the solid node equation numbers
16928  bool do_halos = true;
16929  bool do_external_halos = false;
16930  copy_haloed_eqn_numbers_helper(do_halos, do_external_halos);
16931 
16933  {
16934  t_end = TimingHelpers::timer();
16935  oomph_info << "Time for copy_haloed_eqn_numbers_helper for halos: "
16936  << t_end - t_start << std::endl;
16937  t_start = TimingHelpers::timer();
16938  }
16939 
16940  // Now do external halo stuff
16941  do_halos = false;
16942  do_external_halos = true;
16943  copy_haloed_eqn_numbers_helper(do_halos, do_external_halos);
16944 
16946  {
16947  t_end = TimingHelpers::timer();
16948  oomph_info
16949  << "Time for copy_haloed_eqn_numbers_helper for external halos: "
16950  << t_end - t_start << std::endl;
16951  t_start = TimingHelpers::timer();
16952  }
16953 
16954  // Now the global equation numbers have been updated.
16955  //---------------------------------------------------
16956  // Setup the local equation numbers again.
16957  //----------------------------------------
16958  if (assign_local_eqn_numbers)
16959  {
16960  // Loop over the submeshes: Note we need to call the submeshes' own
16961  // assign_*_eqn_number() otherwise we miss additional functionality
16962  // that is implemented (e.g.) in SolidMeshes!
16963  unsigned n_sub_mesh = nsub_mesh();
16964  if (n_sub_mesh == 0)
16965  {
16967  }
16968  else
16969  {
16970  for (unsigned i = 0; i < n_sub_mesh; i++)
16971  {
16973  }
16974  }
16975  }
16976 
16978  {
16979  t_end = TimingHelpers::timer();
16980  oomph_info << "Time for assign_local_eqn_numbers in sync: "
16981  << t_end - t_start << std::endl;
16982  t_start = TimingHelpers::timer();
16983  }
16984 
16985  // wait for the sends to complete
16986  if (n_send > 0)
16987  {
16988  Vector<MPI_Status> send_status(n_send);
16989  MPI_Waitall(n_send, &send_req[0], &send_status[0]);
16990  }
16991 
16993  {
16994  t_end = TimingHelpers::timer();
16995  oomph_info << "Time for waitall: " << t_end - t_start << std::endl;
16996  t_start = TimingHelpers::timer();
16997  }
16998 
16999  // build the Dof distribution pt
17000  Dof_distribution_pt->build(Communicator_pt, my_eqn_num_base, my_n_eqn);
17001 
17002  // and return the total number of equations in the problem
17003  return (long)Dof_distribution_pt->nrow();
17004  }
17005 
17006 
17007  //=======================================================================
17008  /// A private helper function to
17009  /// copy the haloed equation numbers into the halo equation numbers,
17010  /// either for the problem's one and only mesh or for all of its
17011  /// submeshes. Bools control if we deal with data associated with
17012  /// external halo/ed elements/nodes or the "normal" halo/ed ones.
17013  //===================================================================
17014  void Problem::copy_haloed_eqn_numbers_helper(const bool& do_halos,
17015  const bool& do_external_halos)
17016  {
17017  // Do we have submeshes?
17018  unsigned n_mesh_loop = 1;
17019  unsigned nmesh = nsub_mesh();
17020  if (nmesh > 0)
17021  {
17022  n_mesh_loop = nmesh;
17023  }
17024 
17025  // Storage for number of processors and current processor
17026  int n_proc = this->communicator_pt()->nproc();
17027 
17028  // If only one processor then return
17029  if (n_proc == 1)
17030  {
17031  return;
17032  }
17033  int my_rank = this->communicator_pt()->my_rank();
17034 
17035  // Storage for number of data to be sent to each processor
17036  Vector<int> send_n(n_proc, 0);
17037  // Storage for all equation numbers to be sent to all processors
17038  Vector<long> send_data;
17039  // Start location within send_data for data to be sent to each processor
17040  Vector<int> send_displacement(n_proc, 0);
17041 
17042 
17043  // Loop over all processors whose eqn numbers are to be updated
17044  for (int rank = 0; rank < n_proc; rank++)
17045  {
17046  // Set the displacement of the current processor in the loop
17047  send_displacement[rank] = send_data.size();
17048 
17049  // If I'm not the processor whose halo eqn numbers are updated,
17050  // some of my nodes may be haloed: Stick their
17051  // eqn numbers into the vector
17052  if (rank != my_rank)
17053  {
17054  // Deal with sub-meshes one-by-one if required
17055  Mesh* my_mesh_pt = 0;
17056 
17057  // Loop over submeshes
17058  for (unsigned imesh = 0; imesh < n_mesh_loop; imesh++)
17059  {
17060  if (nmesh == 0)
17061  {
17062  my_mesh_pt = mesh_pt();
17063  }
17064  else
17065  {
17066  my_mesh_pt = mesh_pt(imesh);
17067  }
17068 
17069  if (do_halos)
17070  {
17071  // Add equation numbers for each haloed node
17072  unsigned n_nod = my_mesh_pt->nhaloed_node(rank);
17073  for (unsigned n = 0; n < n_nod; n++)
17074  {
17075  my_mesh_pt->haloed_node_pt(rank, n)->add_eqn_numbers_to_vector(
17076  send_data);
17077  }
17078 
17079  // Add the equation numbers associated with internal data
17080  // in the haloed elements
17081  Vector<GeneralisedElement*> haloed_elem_pt =
17082  my_mesh_pt->haloed_element_pt(rank);
17083  unsigned nelem_haloed = haloed_elem_pt.size();
17084  for (unsigned e = 0; e < nelem_haloed; e++)
17085  {
17086  haloed_elem_pt[e]->add_internal_eqn_numbers_to_vector(send_data);
17087  }
17088  }
17089 
17090  if (do_external_halos)
17091  {
17092  // Add equation numbers associated with external haloed nodes
17093  unsigned n_ext_nod = my_mesh_pt->nexternal_haloed_node(rank);
17094  for (unsigned n = 0; n < n_ext_nod; n++)
17095  {
17096  my_mesh_pt->external_haloed_node_pt(rank, n)
17097  ->add_eqn_numbers_to_vector(send_data);
17098  }
17099 
17100  // Add the equation numbers associated with internal data in
17101  // each external haloed element
17102  unsigned next_elem_haloed =
17103  my_mesh_pt->nexternal_haloed_element(rank);
17104  for (unsigned e = 0; e < next_elem_haloed; e++)
17105  {
17106  // how many internal data values for this element?
17107  my_mesh_pt->external_haloed_element_pt(rank, e)
17109  }
17110  }
17111 
17112  } // end of loop over meshes
17113  }
17114 
17115  // Find the number of data added to the vector by this processor
17116  send_n[rank] = send_data.size() - send_displacement[rank];
17117  }
17118 
17119  // Storage for the number of data to be received from each processor
17120  Vector<int> receive_n(n_proc, 0);
17121 
17122  // Communicate all numbers of data to be sent between all processors
17123  MPI_Alltoall(&send_n[0],
17124  1,
17125  MPI_INT,
17126  &receive_n[0],
17127  1,
17128  MPI_INT,
17129  this->communicator_pt()->mpi_comm());
17130 
17131  // We now prepare the data to be received
17132  // by working out the displacements from the received data
17133  Vector<int> receive_displacement(n_proc, 0);
17134  int receive_data_count = 0;
17135  for (int rank = 0; rank < n_proc; ++rank)
17136  {
17137  // Displacement is number of data received so far
17138  receive_displacement[rank] = receive_data_count;
17139  receive_data_count += receive_n[rank];
17140  }
17141 
17142  // Now resize the receive buffer
17143  // Make sure that it has a size of at least one
17144  if (receive_data_count == 0)
17145  {
17146  ++receive_data_count;
17147  }
17148  Vector<long> receive_data(receive_data_count);
17149 
17150  // Make sure that the send buffer has size at least one
17151  // so that we don't get a segmentation fault
17152  if (send_data.size() == 0)
17153  {
17154  send_data.resize(1);
17155  }
17156 
17157  // Now send the data between all the processors
17158  MPI_Alltoallv(&send_data[0],
17159  &send_n[0],
17160  &send_displacement[0],
17161  MPI_LONG,
17162  &receive_data[0],
17163  &receive_n[0],
17164  &receive_displacement[0],
17165  MPI_LONG,
17166  this->communicator_pt()->mpi_comm());
17167 
17168 
17169  // Loop over all other processors to receive their
17170  // eqn numbers
17171  for (int send_rank = 0; send_rank < n_proc; send_rank++)
17172  {
17173  // Don't do anything for the processor corresponding to the
17174  // current processor or if no data were received from this processor
17175  if ((send_rank != my_rank) && (receive_n[send_rank] != 0))
17176  {
17177  // Counter for the data within the large array
17178  unsigned count = receive_displacement[send_rank];
17179 
17180  // Deal with sub-meshes one-by-one if required
17181  Mesh* my_mesh_pt = 0;
17182 
17183  // Loop over submeshes
17184  for (unsigned imesh = 0; imesh < n_mesh_loop; imesh++)
17185  {
17186  if (nmesh == 0)
17187  {
17188  my_mesh_pt = mesh_pt();
17189  }
17190  else
17191  {
17192  my_mesh_pt = mesh_pt(imesh);
17193  }
17194 
17195  if (do_halos)
17196  {
17197  // How many of my nodes are halos whose non-halo counter
17198  // parts live on processor send_rank?
17199  unsigned n_nod = my_mesh_pt->nhalo_node(send_rank);
17200  for (unsigned n = 0; n < n_nod; n++)
17201  {
17202  // Generalise to variable number of values per node
17203  my_mesh_pt->halo_node_pt(send_rank, n)
17204  ->read_eqn_numbers_from_vector(receive_data, count);
17205  }
17206 
17207  // Get number of halo elements whose non-halo is on
17208  // process send_rank
17209  Vector<GeneralisedElement*> halo_elem_pt =
17210  my_mesh_pt->halo_element_pt(send_rank);
17211  unsigned nelem_halo = halo_elem_pt.size();
17212  for (unsigned e = 0; e < nelem_halo; e++)
17213  {
17214  halo_elem_pt[e]->read_internal_eqn_numbers_from_vector(
17215  receive_data, count);
17216  }
17217  }
17218 
17219  if (do_external_halos)
17220  {
17221  // How many of my nodes are external halos whose external non-halo
17222  // counterparts live on processor send_rank?
17223  unsigned n_ext_nod = my_mesh_pt->nexternal_halo_node(send_rank);
17224  for (unsigned n = 0; n < n_ext_nod; n++)
17225  {
17226  my_mesh_pt->external_halo_node_pt(send_rank, n)
17227  ->read_eqn_numbers_from_vector(receive_data, count);
17228  }
17229 
17230  // Get number of external halo elements whose external haloed
17231  // counterpart is on process send_rank
17232  unsigned next_elem_halo =
17233  my_mesh_pt->nexternal_halo_element(send_rank);
17234  for (unsigned e = 0; e < next_elem_halo; e++)
17235  {
17236  my_mesh_pt->external_halo_element_pt(send_rank, e)
17237  ->read_internal_eqn_numbers_from_vector(receive_data, count);
17238  }
17239  }
17240 
17241  } // end of loop over meshes
17242  }
17243  } // End of loop over processors
17244  }
17245 
17246  //==========================================================================
17247  /// Balance the load of a (possibly non-uniformly refined) problem that has
17248  /// already been distributed, by re-distributing elements over the processors.
17249  /// Produce explicit stats of load balancing process if boolean, report_stats,
17250  /// is set to true and doc various bits of data (mainly for debugging)
17251  /// in directory specified by DocInfo object.
17252  //==========================================================================
17254  DocInfo& doc_info,
17255  const bool& report_stats,
17256  const Vector<unsigned>& input_target_domain_for_local_non_halo_element)
17257  {
17258  double start_t = TimingHelpers::timer();
17259 
17260  // Number of processes
17261  const unsigned n_proc = this->communicator_pt()->nproc();
17262 
17263  // Don't do anything if this is a single-process job
17264  if (n_proc == 1)
17265  {
17266  if (report_stats)
17267  {
17268  std::ostringstream warn_message;
17269  warn_message << "WARNING: You've tried to load balance a problem over\n"
17270  << "only one processor: ignoring your request.\n";
17271  OomphLibWarning(warn_message.str(),
17272  "Problem::load_balance()",
17273  OOMPH_EXCEPTION_LOCATION);
17274  }
17275  }
17276  // Multiple processors
17277  else
17278  {
17279  // This will only work if the problem has already been distributed
17281  {
17282  // Throw an error
17283  std::ostringstream error_stream;
17284  error_stream << "You have called Problem::load_balance()\n"
17285  << "on a non-distributed problem. This doesn't\n"
17286  << "make sense -- go distribute your problem first."
17287  << std::endl;
17288  throw OomphLibError(
17289  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
17290  }
17291 
17292  // Timings
17293  double t_start = 0.0;
17294  double t_metis = 0.0;
17295  double t_partition = 0.0;
17296  double t_distribute = 0.0;
17297  double t_refine = 0.0;
17298  double t_copy_solution = 0.0;
17299 
17300  if (report_stats)
17301  {
17302  t_start = TimingHelpers::timer();
17303  }
17304 
17305 
17306 #ifdef PARANOID
17307  unsigned old_ndof = ndof();
17308 #endif
17309 
17310  // Store pointers to the old mesh(es) so we retain a handle
17311  //---------------------------------------------------------
17312  // to them for deletion
17313  //---------------------
17314  Vector<Mesh*> old_mesh_pt;
17315  unsigned n_mesh = nsub_mesh();
17316  if (n_mesh == 0)
17317  {
17318  // Resize the container
17319  old_mesh_pt.resize(1);
17320  old_mesh_pt[0] = mesh_pt();
17321  }
17322  else
17323  {
17324  // Resize the container
17325  old_mesh_pt.resize(n_mesh);
17326  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
17327  {
17328  old_mesh_pt[i_mesh] = mesh_pt(i_mesh);
17329  }
17330  }
17331 
17332 
17333  // Partition the global mesh in its current state
17334  //-----------------------------------------------
17335 
17336  // target_domain_for_local_non_halo_element[e] contains the number
17337  // of the domain [0,1,...,nproc-1] to which non-halo element e on THE
17338  // CURRENT PROCESSOR ONLY has been assigned. The order of the non-halo
17339  // elements is the same as in the Problem's mesh, with the halo
17340  // elements being skipped.
17341  Vector<unsigned> target_domain_for_local_non_halo_element;
17342 
17343  // Do any of the processors want to go through externally imposed
17344  // partitioning? If so, we'd better do it here too (even if the processor
17345  // is empty, e.g. following a restart on a larger number of procs) or
17346  // we hang.
17347  unsigned local_ntarget =
17348  input_target_domain_for_local_non_halo_element.size();
17349  unsigned global_ntarget = 0;
17350  MPI_Allreduce(&local_ntarget,
17351  &global_ntarget,
17352  1,
17353  MPI_UNSIGNED,
17354  MPI_MAX,
17355  Communicator_pt->mpi_comm());
17356 
17357  // External prescribed partitioning
17358  if (global_ntarget > 0)
17359  {
17360  target_domain_for_local_non_halo_element =
17361  input_target_domain_for_local_non_halo_element;
17362  }
17363  else
17364  {
17365  // Metis does not always produce repeatable results which is
17366  // a disaster for validation runs -- this bypasses metis and
17367  // comes up with a stupid but repeatable partioning.
17369  {
17370  // Bypass METIS to perform the partitioning
17371  unsigned objective = 0;
17372  bool bypass_metis = true;
17374  this,
17375  objective,
17376  target_domain_for_local_non_halo_element,
17377  bypass_metis);
17378  }
17379  else
17380  {
17381  // Use METIS to perform the partitioning
17382  unsigned objective = 0;
17384  this, objective, target_domain_for_local_non_halo_element);
17385  }
17386  }
17387 
17388  if (report_stats)
17389  {
17390  t_metis = TimingHelpers::timer();
17391  }
17392 
17393  // Setup map linking element with target domain
17394  std::map<GeneralisedElement*, unsigned>
17395  target_domain_for_local_non_halo_element_map;
17396  unsigned n_elem = mesh_pt()->nelement();
17397  unsigned count_non_halo_el = 0;
17398  for (unsigned e = 0; e < n_elem; e++)
17399  {
17400  GeneralisedElement* el_pt = mesh_pt()->element_pt(e);
17401  if (!el_pt->is_halo())
17402  {
17403  target_domain_for_local_non_halo_element_map[el_pt] =
17404  target_domain_for_local_non_halo_element[count_non_halo_el];
17405  count_non_halo_el++;
17406  }
17407  }
17408 
17409  // Load balancing is equivalent to distribution so call the
17410  // appropriate "actions before". NOTE: This acts on the
17411  // current, refined, distributed, etc. problem object
17412  // before it's being wiped. This step is therefore not
17413  // a duplicate of the call below, which acts on the
17414  // new, not-yet refined, distributed etc. problem!
17416 
17417  // Re-setup target domains for remaining elements (FaceElements
17418  // are likely to have been stripped out in actions_before_distribute()
17419  n_elem = mesh_pt()->nelement();
17420  target_domain_for_local_non_halo_element.clear();
17421  target_domain_for_local_non_halo_element.reserve(n_elem);
17422  count_non_halo_el = 0;
17423  for (unsigned e = 0; e < n_elem; e++)
17424  {
17425  GeneralisedElement* el_pt = mesh_pt()->element_pt(e);
17426  if (!el_pt->is_halo())
17427  {
17428  target_domain_for_local_non_halo_element.push_back(
17429  target_domain_for_local_non_halo_element_map[el_pt]);
17430  }
17431  } // for (e < n_elem)
17432 
17433  // Re-setup the number of sub-meshes since some of them may have
17434  // been stripped out in actions_before_distribute(), but save the
17435  // number of old sub-meshes
17436  const unsigned n_old_sub_meshes = n_mesh;
17437  n_mesh = nsub_mesh();
17438 
17439  // Now get the target domains for each of the submeshes, we only
17440  // get the target domains for the nonhalo elements
17441  Vector<Vector<unsigned>> target_domain_for_local_non_halo_element_submesh(
17442  n_mesh);
17443  // If we have no sub-meshes then we do not need to copy the target areas
17444  // of the submeshes
17445  if (n_mesh != 0)
17446  {
17447  // Counter to copy the target domains from the global vector
17448  unsigned count_td = 0;
17449  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
17450  {
17451  // Get the number of elements (considering halo elements)
17452  const unsigned nsub_ele = mesh_pt(i_mesh)->nelement();
17453  // Now copy that number of data from the global target domains
17454  for (unsigned i = 0; i < nsub_ele; i++)
17455  {
17456  // Get the element
17457  GeneralisedElement* ele_pt = mesh_pt(i_mesh)->element_pt(i);
17458  // ... and check if it is a nonhalo element
17459  if (!ele_pt->is_halo())
17460  {
17461  // Get the target domain for the current element
17462  const unsigned target_domain =
17463  target_domain_for_local_non_halo_element[count_td++];
17464  // Add the target domain for the nonhalo element in the
17465  // submesh
17466  target_domain_for_local_non_halo_element_submesh[i_mesh]
17467  .push_back(target_domain);
17468  } // if (!ele_pt->is_halo())
17469  } // for (i < nsub_ele)
17470  } // for (imesh < n_mesh)
17471 
17472 #ifdef PARANOID
17473  // Check that the total number of copied data be the same as the
17474  // total number of nonhalo elements in the (sub)-mesh(es)
17475  const unsigned ntarget_domain =
17476  target_domain_for_local_non_halo_element.size();
17477  if (count_td != ntarget_domain)
17478  {
17479  std::ostringstream error_stream;
17480  error_stream
17481  << "The number of nonhalo elements (" << count_td
17482  << ") found in (all)\n"
17483  << "the (sub)-mesh(es) is different from the number of target "
17484  "domains\n"
17485  << "(" << ntarget_domain << ") for the nonhalo elements.\n"
17486  << "Please ensure that you called the rebuild_global_mesh() method "
17487  << "after the\npossible deletion of FaceElements in "
17488  << "actions_before_distribute()!!!\n\n";
17489  throw OomphLibError(error_stream.str(),
17490  "Problem::load_balance()",
17491  OOMPH_EXCEPTION_LOCATION);
17492  } // if (count_td != ntarget_domain)
17493 #endif
17494 
17495  } // if (n_mesh != 0)
17496 
17497  // Check if we have different type of submeshes (unstructured
17498  // and/or structured). Identify to which type each submesh belongs.
17499  // If we have only one mesh then identify to which type that mesh
17500  // belongs.
17501 
17502  // The load balancing strategy acts in the structured meshes and
17503  // then acts in the unstructured meshes
17504 
17505  // Vector to temporaly store pointers to unstructured meshes
17506  // (TriangleMeshBase)
17507  Vector<TriangleMeshBase*> unstructured_mesh_pt;
17508  std::vector<bool> is_unstructured_mesh;
17509 
17510  // Flag to indicate that there are unstructured meshes as part of
17511  // the problem
17512  bool are_there_unstructured_meshes = false;
17513 
17514  // We have only one mesh
17515  if (n_mesh == 0)
17516  {
17517  // Check if it is a TriangleMeshBase mesh
17518  if (TriangleMeshBase* tri_mesh_pt =
17519  dynamic_cast<TriangleMeshBase*>(old_mesh_pt[0]))
17520  {
17521  // Add the pointer to the unstructured meshes container
17522  unstructured_mesh_pt.push_back(tri_mesh_pt);
17523  // Indicate that it is an unstructured mesh
17524  is_unstructured_mesh.push_back(true);
17525  // Indicate that there are unstructured meshes as part of the
17526  // problem
17527  are_there_unstructured_meshes = true;
17528  }
17529  else
17530  {
17531  // Add the pointer to the unstructured meshes container (null
17532  // pointer)
17533  unstructured_mesh_pt.push_back(tri_mesh_pt);
17534  // Indicate that it is not an unstructured mesh
17535  is_unstructured_mesh.push_back(false);
17536  }
17537  } // if (n_mesh == 0)
17538  else // We have sub-meshes
17539  {
17540  // Check which sub-meshes are unstructured meshes (work with the
17541  // old sub-meshes number)
17542  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
17543  {
17544  // Is it a TriangleMeshBase mesh
17545  if (TriangleMeshBase* tri_mesh_pt =
17546  dynamic_cast<TriangleMeshBase*>(old_mesh_pt[i_mesh]))
17547  {
17548  // Add the pointer to the unstructured meshes container
17549  unstructured_mesh_pt.push_back(tri_mesh_pt);
17550  // Indicate that it is an unstructured mesh
17551  is_unstructured_mesh.push_back(true);
17552  // Indicate that there are unstructured meshes as part of the
17553  // problem
17554  are_there_unstructured_meshes = true;
17555  }
17556  else
17557  {
17558  // Add the pointer to the unstructured meshes container (null
17559  // pointer)
17560  unstructured_mesh_pt.push_back(tri_mesh_pt);
17561  // Indicate that it is not an unstructured mesh
17562  is_unstructured_mesh.push_back(false);
17563  }
17564  } // for (i_mesh < n_mesh)
17565  } // else if (n_mesh == 0) // We have sub-meshes
17566 
17567  // Extract data to be sent to various processors after the
17568  //--------------------------------------------------------
17569  // problem has been rebuilt/re-distributed
17570  //----------------------------------------
17571 
17572  // Storage for number of data to be sent to each processor
17573  Vector<int> send_n(n_proc, 0);
17574 
17575  // Storage for all values to be sent to all processors
17576  Vector<double> send_data;
17577 
17578  // Start location within send_data for data to be sent to each processor
17579  Vector<int> send_displacement(n_proc, 0);
17580 
17581  // Old and new domains for each base element (available for all, for
17582  // convenience)
17583  Vector<unsigned> old_domain_for_base_element;
17584  Vector<unsigned> new_domain_for_base_element;
17585 
17586  // Flat-packed refinement info, labeled by id of locally
17587  // available root elements
17588  std::map<unsigned, Vector<unsigned>> flat_packed_refinement_info_for_root;
17589 
17590  // Max. level of refinement
17591  unsigned max_refinement_level_overall = 0;
17592 
17593  // Prepare the input for the get_data...() method, only copy the
17594  // data from the structured meshes, TreeBaseMesh meshes
17596  target_domain_for_local_non_halo_element_in_structured_mesh;
17597  if (n_mesh == 0)
17598  {
17599  // Check if the mesh is an structured mesh
17600  if (!is_unstructured_mesh[0])
17601  {
17602  const unsigned nele_mesh =
17603  target_domain_for_local_non_halo_element.size();
17604  for (unsigned e = 0; e < nele_mesh; e++)
17605  {
17606  const unsigned target_domain =
17607  target_domain_for_local_non_halo_element[e];
17608  target_domain_for_local_non_halo_element_in_structured_mesh
17609  .push_back(target_domain);
17610  } // for (e < nele_mesh)
17611  } // if (!is_unstructured_mesh[0])
17612  } // if (n_mesh == 0)
17613  else
17614  {
17615  // Copy the target domains from the structured meshes only
17616  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
17617  {
17618  // Check if the mesh is an structured mesh
17619  if (!is_unstructured_mesh[i_mesh])
17620  {
17621  const unsigned nele_sub_mesh =
17622  target_domain_for_local_non_halo_element_submesh[i_mesh].size();
17623  for (unsigned e = 0; e < nele_sub_mesh; e++)
17624  {
17625  const unsigned target_domain =
17626  target_domain_for_local_non_halo_element_submesh[i_mesh][e];
17627  target_domain_for_local_non_halo_element_in_structured_mesh
17628  .push_back(target_domain);
17629  } // for (e < nele_sub_mesh)
17630  } // if (!is_triangle_mesh_base[i_mesh])
17631  } // for (i_mesh < n_mesh)
17632  } // else if (n_mesh == 0)
17633 
17634  // Extract data from current problem
17635  // sorted into data to be sent to various processors after
17636  // rebuilding the meshes in a load-balanced form
17638  target_domain_for_local_non_halo_element_in_structured_mesh,
17639  send_n,
17640  send_data,
17641  send_displacement,
17642  old_domain_for_base_element,
17643  new_domain_for_base_element,
17644  max_refinement_level_overall);
17645 
17646  // Extract flat-packed refinement pattern
17648  old_domain_for_base_element,
17649  new_domain_for_base_element,
17650  max_refinement_level_overall,
17651  flat_packed_refinement_info_for_root);
17652 
17653  if (report_stats)
17654  {
17655  t_partition = TimingHelpers::timer();
17656  oomph_info << "CPU for partition calculation for roots: "
17657  << t_partition - t_metis << std::endl;
17658  }
17659 
17660 
17661  // Flush and delete old submeshes and null the global mesh
17662  //--------------------------------------------------------
17663  // and rebuild the new (not yet distributed, refined etc.) mesh
17664  //-------------------------------------------------------------
17665  // that will be distributed in the new, improved way determined
17666  //-------------------------------------------------------------
17667  // by METIS
17668  //---------
17669  Vector<unsigned> pruned_refinement_level(
17670  std::max(int(n_old_sub_meshes), 1));
17671  if (n_mesh == 0)
17672  {
17673  pruned_refinement_level[0] = 0;
17674  TreeBasedRefineableMeshBase* ref_mesh_pt =
17675  dynamic_cast<TreeBasedRefineableMeshBase*>(old_mesh_pt[0]);
17676  if (ref_mesh_pt != 0)
17677  {
17678  pruned_refinement_level[0] =
17679  ref_mesh_pt->uniform_refinement_level_when_pruned();
17680  }
17681 
17682  // If the mesh is an unstructured mesh (TriangleMeshBase mesh)
17683  // then we should not delete it since the load balance strategy
17684  // requires the mesh
17685 
17686  // Delete the mesh if it is not an unstructured mesh
17687  if (!is_unstructured_mesh[0])
17688  {
17689  delete old_mesh_pt[0];
17690  old_mesh_pt[0] = 0;
17691  } // if (!is_unstructured_mesh[0])
17692  } // if (n_mesh==0)
17693  else
17694  {
17695  // Loop over the number of old meshes (required to delete the
17696  // pointers of structured meshes in the old_mesh_pt structure)
17697  for (unsigned i_mesh = 0; i_mesh < n_old_sub_meshes; i_mesh++)
17698  {
17699  pruned_refinement_level[i_mesh] = 0;
17700  TreeBasedRefineableMeshBase* ref_mesh_pt =
17701  dynamic_cast<TreeBasedRefineableMeshBase*>(old_mesh_pt[i_mesh]);
17702  if (ref_mesh_pt != 0)
17703  {
17704  pruned_refinement_level[i_mesh] =
17705  ref_mesh_pt->uniform_refinement_level_when_pruned();
17706  }
17707 
17708  // If the mesh is an unstructured mesh (TriangleMeshBase mesh)
17709  // then we should NOT delete it since the load balance strategy
17710  // requires the mesh
17711 
17712  // Delete the mesh if it is not an unstructured mesh
17713  if (!is_unstructured_mesh[i_mesh])
17714  {
17715  delete old_mesh_pt[i_mesh];
17716  old_mesh_pt[i_mesh] = 0;
17717  } // if (!is_unstructured_mesh[i_mesh])
17718 
17719  } // for (i_mesh<n_mesh)
17720 
17721  // Empty storage for sub-meshes
17722  flush_sub_meshes();
17723 
17724  // Flush the storage for nodes and elements in compound mesh
17725  // (they've already been deleted in the sub-meshes)
17727 
17728  // Kill
17729  delete mesh_pt();
17730  mesh_pt() = 0;
17731  } // else if (n_mesh==0)
17732 
17733  bool some_mesh_has_been_pruned = false;
17734  unsigned n = pruned_refinement_level.size();
17735  for (unsigned i = 0; i < n; i++)
17736  {
17737  if (pruned_refinement_level[i] > 0) some_mesh_has_been_pruned = true;
17738  }
17739 
17740  // (Re-)build the new mesh(es) -- this must get the problem into the
17741  // state it was in when it was first distributed!
17742  build_mesh();
17743 
17744  // Has one of the meshes been pruned; if so refine to the
17745  // common refinement level
17746  if (some_mesh_has_been_pruned)
17747  {
17748  // Do actions before adapt
17750 
17751  // Re-assign number of submeshes -- when this was first
17752  // set, the problem may have had face meshes that have now
17753  // disappeared.
17754  n_mesh = nsub_mesh();
17755 
17756  // Now adapt meshes manually
17757  if (n_mesh == 0)
17758  {
17759  TreeBasedRefineableMeshBase* ref_mesh_pt =
17760  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt());
17761  if (ref_mesh_pt != 0)
17762  {
17763  // Get min and max refinement level
17764  unsigned local_min_ref = 0;
17765  unsigned local_max_ref = 0;
17766  ref_mesh_pt->get_refinement_levels(local_min_ref, local_max_ref);
17767 
17768  // Reconcile between processors: If (e.g. following
17769  // distribution/pruning) the mesh has no elements on this
17770  // processor) then ignore its contribution to the poll of
17771  // max/min refinement levels
17772  int int_local_min_ref = local_min_ref;
17773  if (ref_mesh_pt->nelement() == 0)
17774  {
17775  int_local_min_ref = INT_MAX;
17776  }
17777  int int_min_ref = 0;
17778  MPI_Allreduce(&int_local_min_ref,
17779  &int_min_ref,
17780  1,
17781  MPI_INT,
17782  MPI_MIN,
17783  Communicator_pt->mpi_comm());
17784 
17785  // Overall min refinement level over all meshes
17786  unsigned min_ref = unsigned(int_min_ref);
17787 
17788  // Refine as many times as required to get refinement up to
17789  // uniform refinement level after last prune
17790  unsigned nref = pruned_refinement_level[0] - min_ref;
17791  oomph_info << "Refining one-and-only mesh uniformly " << nref
17792  << " times\n";
17793  for (unsigned i = 0; i < nref; i++)
17794  {
17795  ref_mesh_pt->refine_uniformly();
17796  }
17797  }
17798  }
17799  else
17800  {
17801  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
17802  {
17803  TreeBasedRefineableMeshBase* ref_mesh_pt =
17804  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(i_mesh));
17805  if (ref_mesh_pt != 0)
17806  {
17807  // Get min and max refinement level
17808  unsigned local_min_ref = 0;
17809  unsigned local_max_ref = 0;
17810  ref_mesh_pt->get_refinement_levels(local_min_ref, local_max_ref);
17811 
17812  // Reconcile between processors: If (e.g. following
17813  // distribution/pruning) the mesh has no elements on this
17814  // processor) then ignore its contribution to the poll of
17815  // max/min refinement levels
17816  int int_local_min_ref = local_min_ref;
17817  if (ref_mesh_pt->nelement() == 0)
17818  {
17819  int_local_min_ref = INT_MAX;
17820  }
17821  int int_min_ref = 0;
17822  MPI_Allreduce(&int_local_min_ref,
17823  &int_min_ref,
17824  1,
17825  MPI_INT,
17826  MPI_MIN,
17827  Communicator_pt->mpi_comm());
17828 
17829  // Overall min refinement level over all meshes
17830  unsigned min_ref = unsigned(int_min_ref);
17831 
17832  // Refine as many times as required to get refinement up to
17833  // uniform refinement level after last prune
17834  unsigned nref = pruned_refinement_level[i_mesh] - min_ref;
17835  oomph_info << "Refining sub-mesh " << i_mesh << " uniformly "
17836  << nref << " times\n";
17837  for (unsigned i = 0; i < nref; i++)
17838  {
17839  ref_mesh_pt->refine_uniformly();
17840  }
17841  }
17842  }
17843  // Rebuild the global mesh
17845  }
17846 
17847  // Do actions after adapt
17849 
17850  // Re-assign number of submeshes -- when this was first
17851  // set, the problem may have had face meshes that have now
17852  // disappeared.
17853  n_mesh = nsub_mesh();
17854  } // if (some_mesh_has_been_pruned)
17855 
17856 
17857  // Perform any actions before distribution but now for the new mesh
17858  // NOTE: This does NOT replicate the actions_before_distribute()
17859  // call made above for the previous mesh!
17861 
17862  // Do some book-keeping
17863  //---------------------
17864 
17865  // Re-assign number of submeshes -- when this was first
17866  // set, the problem may have had face meshes that have now
17867  // disappeared.
17868  n_mesh = nsub_mesh();
17869 
17870  // The submeshes, if they exist, need to know their own element
17871  // domains.
17872  // NOTE: This vector only stores the target domains or the
17873  // element partition for structured meshes
17874  Vector<Vector<unsigned>> submesh_element_partition(n_mesh);
17875  if (n_mesh != 0)
17876  {
17877  unsigned count = 0;
17878  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
17879  {
17880  // Only work with structured meshes
17881  if (!is_unstructured_mesh[i_mesh])
17882  {
17883  // Get the number of element in the mesh
17884  const unsigned nsub_ele = mesh_pt(i_mesh)->nelement();
17885  submesh_element_partition[i_mesh].resize(nsub_ele);
17886  for (unsigned e = 0; e < nsub_ele; e++)
17887  {
17888  submesh_element_partition[i_mesh][e] =
17889  new_domain_for_base_element[count++];
17890  } // for (e<nsub_elem)
17891  } // if (sub_mesh_pt!=0)
17892  } // for (i_mesh<n_mesh)
17893 
17894 #ifdef PARANOID
17895  const unsigned nnew_domain_for_base_element =
17896  new_domain_for_base_element.size();
17897  if (count != nnew_domain_for_base_element)
17898  {
17899  std::ostringstream error_stream;
17900  error_stream
17901  << "The number of READ target domains for nonhalo elements\n"
17902  << " is (" << count << "), but the number of target domains for\n"
17903  << "nonhalo elements is (" << nnew_domain_for_base_element
17904  << ")!\n";
17905  throw OomphLibError(error_stream.str(),
17906  "Problem::load_balance()",
17907  OOMPH_EXCEPTION_LOCATION);
17908  }
17909 #endif
17910 
17911  } // if (n_mesh!=0)
17912 
17913  // Setup the map between "root" element and number in global mesh
17914  // again
17915 
17916  // This map is only established for structured meshes, then we
17917  // need to check here the type of mesh
17918  if (n_mesh == 0)
17919  {
17920  // Check if the only one mesh is an stuctured mesh
17921  if (!is_unstructured_mesh[0])
17922  {
17923  const unsigned n_ele = mesh_pt()->nelement();
17924  Base_mesh_element_pt.resize(n_ele);
17926  for (unsigned e = 0; e < n_ele; e++)
17927  {
17928  GeneralisedElement* el_pt = mesh_pt()->element_pt(e);
17929  Base_mesh_element_number_plus_one[el_pt] = e + 1;
17930  Base_mesh_element_pt[e] = el_pt;
17931  } // for (e<n_ele)
17932  } // if (!is_triangle_mesh_base[0])
17933  } // if (n_mesh==0)
17934  else
17935  {
17936  // If we have submeshes then we only add those elements that
17937  // belong to structured meshes, but first compute the number of
17938  // total elements in the structured sub-meshes
17939  unsigned nglobal_element = 0;
17940  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
17941  {
17942  // Check if mesh is an structured mesh
17943  if (!is_unstructured_mesh[i_mesh])
17944  {
17945  nglobal_element += mesh_pt(i_mesh)->nelement();
17946  } // if (!is_triangle_mesh_base[i_mesh])
17947  } // for (i_mesh<n_mesh)
17948 
17949  // Once computed the number of elements, then resize the
17950  // structure
17951  Base_mesh_element_pt.resize(nglobal_element);
17953  unsigned counter_base = 0;
17954  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
17955  {
17956  // Check if mesh is a structured mesh
17957  if (!is_unstructured_mesh[i_mesh])
17958  {
17959  const unsigned n_ele = mesh_pt(i_mesh)->nelement();
17960  for (unsigned e = 0; e < n_ele; e++)
17961  {
17962  GeneralisedElement* el_pt = mesh_pt(i_mesh)->element_pt(e);
17963  Base_mesh_element_number_plus_one[el_pt] = counter_base + 1;
17964  Base_mesh_element_pt[counter_base] = el_pt;
17965  // Inrease the global element number
17966  counter_base++;
17967  } // for (e<n_ele)
17968  } // if (!is_triangle_mesh_base[i_mesh])
17969  } // for (i_mesh<n_mesh)
17970 
17971 #ifdef PARANOID
17972  if (counter_base != nglobal_element)
17973  {
17974  std::ostringstream error_stream;
17975  error_stream << "The number of global elements (" << nglobal_element
17976  << ") is not the same as the number of\nadded elements ("
17977  << counter_base << ") to the Base_mesh_element_pt data "
17978  << "structure!!!\n\n";
17979  throw OomphLibError(error_stream.str(),
17980  "Problem::load_balance()",
17981  OOMPH_EXCEPTION_LOCATION);
17982  } // if (counter_base != nglobal_element)
17983 #endif // #ifdef PARANOID
17984 
17985  } // else if (n_mesh==0)
17986 
17987  // Storage for the number of face elements in the base mesh --
17988  // element is identified by number of bulk element and face index
17989  // so we can reconstruct it if and when the FaceElements have been wiped
17990  // in actions_before_distribute().
17991  // NOTE: Not really clear (any more) why this is required. Typically
17992  // FaceElements get wiped in actions_before_distribute() so
17993  // at this point there shouldn't be any of them left.
17994  // This is certainly the case in all our currently existing
17995  // test codes. However, I'm too scared to take this out
17996  // in case it does matter (we're not insisting that FaceElements
17997  // are always removed in actions_before_distribute()...).
17998  std::map<unsigned, std::map<int, unsigned>> face_element_number;
17999  unsigned n_element = mesh_pt()->nelement();
18000  for (unsigned e = 0; e < n_element; e++)
18001  {
18002  FaceElement* face_el_pt =
18003  dynamic_cast<FaceElement*>(mesh_pt()->finite_element_pt(e));
18004  if (face_el_pt != 0)
18005  {
18006 #ifdef PARANOID
18007  std::stringstream info;
18008  info << "================================================\n";
18009  info << "INFO: I've come across a FaceElement while \n";
18010  info << " load-balancing a problem. \n";
18011  info << "================================================\n";
18012  oomph_info << info.str() << std::endl;
18013 #endif
18014  FiniteElement* bulk_elem_pt = face_el_pt->bulk_element_pt();
18015  unsigned e_bulk = Base_mesh_element_number_plus_one[bulk_elem_pt];
18016 #ifdef PARANOID
18017  if (e_bulk == 0)
18018  {
18019  throw OomphLibError("Base_mesh_element_number_plus_one[...]=0",
18020  OOMPH_CURRENT_FUNCTION,
18021  OOMPH_EXCEPTION_LOCATION);
18022  }
18023 #endif
18024  e_bulk -= 1;
18025  int face_index = face_el_pt->face_index();
18026  face_element_number[e_bulk][face_index] = e;
18027  }
18028  }
18029 
18030  // Distribute the (sub)meshes
18031  //---------------------------
18032  Vector<GeneralisedElement*> deleted_element_pt;
18033  if (n_mesh == 0)
18034  {
18035  // Only distribute (load balance strategy) if this is an
18036  // structured mesh
18037  if (!is_unstructured_mesh[0])
18038  {
18039 #ifdef PARANOID
18040  if (mesh_pt()->nelement() != new_domain_for_base_element.size())
18041  {
18042  std::ostringstream error_stream;
18043  error_stream << "Distributing one-and-only mesh containing "
18044  << mesh_pt()->nelement() << " elements with info for "
18045  << new_domain_for_base_element.size() << std::endl;
18046  throw OomphLibError(error_stream.str(),
18047  OOMPH_CURRENT_FUNCTION,
18048  OOMPH_EXCEPTION_LOCATION);
18049  }
18050 #endif
18051 
18052  if (report_stats)
18053  {
18054  oomph_info << "Distributing one and only mesh\n"
18055  << "------------------------------" << std::endl;
18056  }
18057 
18058  // No pre-set distribution from restart that may leave some
18059  // processors empty so no need to overrule deletion of elements
18060  bool overrule_keep_as_halo_element_status = false;
18061 
18062  mesh_pt()->distribute(this->communicator_pt(),
18063  new_domain_for_base_element,
18064  deleted_element_pt,
18065  doc_info,
18066  report_stats,
18067  overrule_keep_as_halo_element_status);
18068 
18069  } // if (!is_unstructured_mesh[0])
18070 
18071  } // if (n_mesh==0)
18072  else // There are submeshes, "distribute" each one separately
18073  {
18074  // Rebuild the mesh only if one of the meshes was modified
18075  bool need_to_rebuild_mesh = false;
18076  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
18077  {
18078  // Perform the load balancing based on distribution in the
18079  // structured meshes only
18080  if (!is_unstructured_mesh[i_mesh])
18081  {
18082  if (report_stats)
18083  {
18084  oomph_info << "Distributing submesh " << i_mesh << " of "
18085  << n_mesh << " in total\n"
18086  << "---------------------------------------------"
18087  << std::endl;
18088  }
18089 
18090  // Set the doc_info number to reflect the submesh
18091  doc_info.number() = i_mesh;
18092 
18093  // No pre-set distribution from restart that may leave some
18094  // processors empty so no need to overrule deletion of elements
18095  bool overrule_keep_as_halo_element_status = false;
18096  mesh_pt(i_mesh)->distribute(this->communicator_pt(),
18097  submesh_element_partition[i_mesh],
18098  deleted_element_pt,
18099  doc_info,
18100  report_stats,
18101  overrule_keep_as_halo_element_status);
18102 
18103  // Set the flag to rebuild the global mesh
18104  need_to_rebuild_mesh = true;
18105 
18106  } // if (!is_unstructured_mesh[i_mesh])
18107 
18108  } // for (i_mesh<n_mesh)
18109 
18110  if (need_to_rebuild_mesh)
18111  {
18112  // Rebuild the global mesh
18114  } // if (need_to_rebuild_mesh)
18115 
18116  } // else if (n_mesh==0)
18117 
18118  // Null out information associated with deleted elements
18119  unsigned n_del = deleted_element_pt.size();
18120  for (unsigned e = 0; e < n_del; e++)
18121  {
18122  GeneralisedElement* el_pt = deleted_element_pt[e];
18123  unsigned old_el_number = Base_mesh_element_number_plus_one[el_pt] - 1;
18125  Base_mesh_element_pt[old_el_number] = 0;
18126  }
18127 
18128  // Has one of the meshes been pruned before distribution? If so
18129  // then prune here now
18130  if (some_mesh_has_been_pruned)
18131  {
18132  Vector<GeneralisedElement*> deleted_element_pt;
18133  if (n_mesh == 0)
18134  {
18135  TreeBasedRefineableMeshBase* ref_mesh_pt =
18136  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt());
18137  if (ref_mesh_pt != 0)
18138  {
18139  ref_mesh_pt->prune_halo_elements_and_nodes(
18140  deleted_element_pt, doc_info, report_stats);
18141  }
18142  }
18143  else
18144  {
18145  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
18146  {
18147  TreeBasedRefineableMeshBase* ref_mesh_pt =
18148  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(i_mesh));
18149  if (ref_mesh_pt != 0)
18150  {
18151  ref_mesh_pt->prune_halo_elements_and_nodes(
18152  deleted_element_pt, doc_info, report_stats);
18153  }
18154  }
18155  // Rebuild the global mesh
18157  }
18158 
18159  // Null out information associated with deleted elements
18160  unsigned n_del = deleted_element_pt.size();
18161  for (unsigned e = 0; e < n_del; e++)
18162  {
18163  GeneralisedElement* el_pt = deleted_element_pt[e];
18164  unsigned old_el_number = Base_mesh_element_number_plus_one[el_pt] - 1;
18166  Base_mesh_element_pt[old_el_number] = 0;
18167  }
18168 
18169  // Setup the map between "root" element and number in global mesh again
18171  }
18172 
18173  if (report_stats)
18174  {
18175  t_distribute = TimingHelpers::timer();
18176  oomph_info << "CPU for build and distribution of new mesh(es): "
18177  << t_distribute - t_partition << std::endl;
18178  }
18179 
18180 
18181  // Send refinement info to other processors
18182  //-----------------------------------------
18183 
18184  // Storage for refinement pattern: Given ID of root element,
18185  // root_element_id, and current refinement level, level, the e-th entry in
18186  // refinement_info_for_root_elements[root_element_id][level][e] is equal
18187 
18188  // to 2 if the e-th element (using the enumeration when the mesh has been
18189  // refined to the level-th level) is to be refined during the next
18190  // refinement; it's 1 if it's not to be refined.
18191  Vector<Vector<Vector<unsigned>>> refinement_info_for_root_elements;
18192 
18193 
18194  // Send refinement information between processors, using flat-packed
18195  // information accumulated earlier
18196  send_refinement_info_helper(old_domain_for_base_element,
18197  new_domain_for_base_element,
18198  max_refinement_level_overall,
18199  flat_packed_refinement_info_for_root,
18200  refinement_info_for_root_elements);
18201 
18202  // Refine each mesh based upon refinement information stored for each root
18203  //------------------------------------------------------------------------
18204  refine_distributed_base_mesh(refinement_info_for_root_elements,
18205  max_refinement_level_overall);
18206 
18207  if (report_stats)
18208  {
18209  t_refine = TimingHelpers::timer();
18210  oomph_info << "CPU for refinement of base mesh: "
18211  << t_refine - t_distribute << std::endl;
18212  }
18213 
18214  // NOTE: The following two calls are important e.g. when
18215  // FaceElements that resize nodes are attached/detached
18216  // after/before adaptation. If we don't attach them
18217  // on the newly built/refined mesh, there isn't enough
18218  // storage for the nodal values that are sent around
18219  // (in a flat-packed format) resulting in total disaster.
18220  // So we attach them first, but then immediatly strip
18221  // them out again because the FaceElements themselves
18222  // will have been stripped out before distribution/adaptation.
18223 
18224  // Do actions after adapt because we have just adapted the mesh.
18226 
18227  // Now strip it back out to get problem into the same state
18228  // it was in when data to be sent was recorded.
18230 
18231  // Send the stored values in each root from the old mesh into the new mesh
18232  //------------------------------------------------------------------------
18234  send_n, send_data, send_displacement);
18235 
18236  // If there are unstructured meshes here we perform the load
18237  // balancing of those meshes
18238  if (are_there_unstructured_meshes)
18239  {
18240  // Delete any storage of external elements and nodes
18242 
18243  if (n_mesh == 0)
18244  {
18245  // Before doing the load balancing delete the mesh created at
18246  // calling build_mesh(), and restore the pointer to the old
18247  // mesh
18248 
18249  // It MUST be an unstructured mesh, otherwise we should not be
18250  // here
18251  if (is_unstructured_mesh[0])
18252  {
18253  // Delete the new created mesh
18254  delete mesh_pt();
18255  // Re-assign the pointer to the old mesh
18256  this->mesh_pt() = old_mesh_pt[0];
18257  } // if (is_unstructured_mesh[0])
18258 #ifdef PARANOID
18259  else
18260  {
18261  std::ostringstream error_stream;
18262  error_stream << "The only one mesh in the problem is not an "
18263  "unstructured mesh,\n"
18264  << "but the flag 'are_there_unstructures_meshes' ("
18265  << are_there_unstructured_meshes
18266  << ") was turned on,\n"
18267  << "this is weird. Please check for any condition "
18268  "that may have\n"
18269  << "turned on this flag!!!!\n\n";
18270  throw OomphLibError(error_stream.str(),
18271  "Problem::load_balance()",
18272  OOMPH_EXCEPTION_LOCATION);
18273  }
18274 #endif
18275 
18276  unstructured_mesh_pt[0]->load_balance(
18277  target_domain_for_local_non_halo_element);
18278  } // if (n_mesh == 0)
18279  else
18280  {
18281  // Before doing the load balancing delete the meshes created
18282  // at calling build_mesh(), and restore the pointer to the
18283  // old meshes
18284  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
18285  {
18286  if (is_unstructured_mesh[i_mesh])
18287  {
18288  // Delete the new created mesh
18289  delete mesh_pt(i_mesh);
18290  // Now point it to nothing
18291  mesh_pt(i_mesh) = 0;
18292  // ... and re-assign the pointer to the old mesh
18293  this->mesh_pt(i_mesh) = old_mesh_pt[i_mesh];
18294  } // if (is_unstructured_mesh[i_mesh])
18295 
18296  } // for (i_mesh<n_mesh)
18297 
18298  // Empty storage for sub-meshes
18299  // flush_sub_meshes();
18300 
18301  // Flush the storage for nodes and elements in compound mesh
18302  // (they've already been deleted in the sub-meshes)
18304 
18305  // Now we can procede with the load balancing thing
18306  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
18307  {
18308  if (is_unstructured_mesh[i_mesh])
18309  {
18310  // Get the number of elements in the "i_mesh" (the old one)
18311  const unsigned n_element = old_mesh_pt[i_mesh]->nelement();
18312 
18313  // Perform the load balancing if there are elements in the
18314  // mesh. We check for this case because the meshes created
18315  // from face elements have been cleaned in
18316  // "actions_before_distribute()"
18317  if (n_element > 0 && is_unstructured_mesh[i_mesh])
18318  {
18319  unstructured_mesh_pt[i_mesh]->load_balance(
18320  target_domain_for_local_non_halo_element_submesh[i_mesh]);
18321  } // if (n_element > 0)
18322  } // if (is_unstructured_mesh[i_mesh)]
18323  } // for (i_mesh < n_mesh)
18324 
18325  // Rebuild the global mesh
18327 
18328  } // else if (n_mesh == 0)
18329 
18330  } // if (are_there_unstructured_meshes)
18331 
18332  if (report_stats)
18333  {
18334  t_copy_solution = TimingHelpers::timer();
18335  oomph_info << "CPU for transferring solution to new mesh(es): "
18336  << t_copy_solution - t_refine << std::endl;
18337  oomph_info << "CPU for load balancing: " << t_copy_solution - t_start
18338  << std::endl;
18339  }
18340 
18341  // Do actions after distribution
18343 
18344  // Re-assign equation numbers
18345 #ifdef PARANOID
18346  unsigned n_dof = assign_eqn_numbers();
18347 #else
18349 #endif
18350 
18351  if (report_stats)
18352  {
18353  oomph_info
18354  << "Total number of elements on this processor after load balance: "
18355  << mesh_pt()->nelement() << std::endl;
18356 
18357  oomph_info << "Number of non-halo elements on this processor after "
18358  "load balance: "
18359  << mesh_pt()->nnon_halo_element() << std::endl;
18360  }
18361 
18362 #ifdef PARANOID
18363  if (n_dof != old_ndof)
18364  {
18365  std::ostringstream error_stream;
18366  error_stream
18367  << "Number of dofs in load_balance() has changed from " << old_ndof
18368  << " to " << n_dof << "\n"
18369  << "Check that you've implemented any necessary "
18370  "actions_before/after\n"
18371  << "adapt/distribute functions, e.g. to pin redundant pressure dofs"
18372  << " etc.\n";
18373  throw OomphLibError(
18374  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
18375  }
18376 #endif
18377  }
18378 
18379  // Finally synchronise all dofs to allow halo check to pass
18381 
18382  double end_t = TimingHelpers::timer();
18383  oomph_info << "Time for load_balance() [sec] : " << end_t - start_t
18384  << std::endl;
18385  }
18386 
18387 
18388  //==========================================================================
18389  /// Send refinement information between processors
18390  //==========================================================================
18392  Vector<unsigned>& old_domain_for_base_element,
18393  Vector<unsigned>& new_domain_for_base_element,
18394  const unsigned& max_refinement_level_overall,
18395  std::map<unsigned, Vector<unsigned>>& flat_packed_refinement_info_for_root,
18396  Vector<Vector<Vector<unsigned>>>& refinement_info_for_root_elements)
18397  {
18398  // Number of processes etc.
18399  const int n_proc = this->communicator_pt()->nproc();
18400  const int my_rank = this->communicator_pt()->my_rank();
18401 
18402  // Make space
18403  unsigned n_base_element = old_domain_for_base_element.size();
18404  refinement_info_for_root_elements.resize(n_base_element);
18405 
18406  // Make space for list of domains that the refinement info
18407  // is to be forwarded to
18408  std::map<unsigned, Vector<unsigned>> halo_domain_of_haloed_base_element;
18409 
18410  // Find out haloed elements in new, redistributed problem
18411  //-------------------------------------------------------
18412 
18413  // halo_domains[e][j] = j-th halo domain associated with (haloed) element e
18414  std::map<unsigned, Vector<unsigned>> halo_domains;
18415 
18416  // Loop over sub meshes
18417  unsigned n_sub_mesh = nsub_mesh();
18418  unsigned max_mesh = std::max(n_sub_mesh, unsigned(1));
18419  for (unsigned i_mesh = 0; i_mesh < max_mesh; i_mesh++)
18420  {
18421  // Choose the right mesh
18422  Mesh* my_mesh_pt = 0;
18423  if (n_sub_mesh == 0)
18424  {
18425  my_mesh_pt = mesh_pt();
18426  }
18427  else
18428  {
18429  my_mesh_pt = mesh_pt(i_mesh);
18430  }
18431 
18432  // Only work with structured meshes
18433  TriangleMeshBase* sub_mesh_pt =
18434  dynamic_cast<TriangleMeshBase*>(mesh_pt(i_mesh));
18435  if (!(sub_mesh_pt != 0))
18436  {
18437  // Loop over processors to find haloed elements -- need to
18438  // send their refinement patterns processors that hold their
18439  // halo counterparts!
18440  for (int p = 0; p < n_proc; p++)
18441  {
18442  Vector<GeneralisedElement*> haloed_elem_pt =
18443  my_mesh_pt->haloed_element_pt(p);
18444  unsigned nhaloed = haloed_elem_pt.size();
18445  for (unsigned h = 0; h < nhaloed; h++)
18446  {
18447  // This element must send its refinement information to processor p
18448  unsigned e = Base_mesh_element_number_plus_one[haloed_elem_pt[h]];
18449 #ifdef PARANOID
18450  if (e == 0)
18451  {
18452  throw OomphLibError("Base_mesh_element_number_plus_one[...]=0",
18453  OOMPH_CURRENT_FUNCTION,
18454  OOMPH_EXCEPTION_LOCATION);
18455  }
18456 #endif
18457  e -= 1;
18458  halo_domains[e].push_back(p);
18459  }
18460  }
18461  } // if (!(sub_mesh_pt!=0))
18462  } // for (i_mesh<max_mesh)
18463 
18464  // Accumulate relevant flat-packed refinement data to be sent to
18465  //--------------------------------------------------------------
18466  // various processors
18467  //-------------------
18468 
18469  // Map to accumulate unsigned data to be sent to each processor
18470  // (map for sparsity)
18471  std::map<unsigned, Vector<unsigned>> data_for_proc;
18472 
18473  // Number of base elements to be sent to specified domain
18474  Vector<unsigned> nbase_elements_for_proc(n_proc, 0);
18475 
18476  // Total number of entries in send vector
18477  unsigned count = 0;
18478 
18479  // Loop over all base elements
18480  //----------------------------
18481  for (unsigned e = 0; e < n_base_element; e++)
18482  {
18483  // Is it one of mine (i.e. was it a non-halo element on this
18484  //----------------------------------------------------------
18485  // processor before re-distribution, and do I therefore hold
18486  //----------------------------------------------------------
18487  // refinement information for it)?
18488  //--------------------------------
18489  if (int(old_domain_for_base_element[e]) == my_rank)
18490  {
18491  // Where does it go?
18492  unsigned new_domain = new_domain_for_base_element[e];
18493 
18494  // Keep counting
18495  nbase_elements_for_proc[new_domain]++;
18496 
18497  // If it stays local, deal with it here
18498  if (int(new_domain) == my_rank)
18499  {
18500  // Record on which other procs/domains the refinement info for
18501  // this element is required because it's haloed.
18502  unsigned nhalo = halo_domains[e].size();
18503  halo_domain_of_haloed_base_element[e].resize(nhalo);
18504  for (unsigned j = 0; j < nhalo; j++)
18505  {
18506  halo_domain_of_haloed_base_element[e][j] = halo_domains[e][j];
18507  }
18508 
18509  // Provide storage for refinement pattern
18510  refinement_info_for_root_elements[e].resize(
18511  max_refinement_level_overall);
18512 
18513 #ifdef PARANOID
18514  // Get number of additional data sent for check
18515  unsigned n_additional_data =
18516  flat_packed_refinement_info_for_root[e].size();
18517 #endif
18518 
18519  // Get number of tree nodes
18520  unsigned n_tree_nodes = flat_packed_refinement_info_for_root[e][0];
18521 
18522  // Counter for entries to be processed locally
18523  unsigned local_count = 1; // (have already processed zero-th entry)
18524 
18525  // Loop over levels and number of nodes in tree
18526  for (unsigned level = 0; level < max_refinement_level_overall;
18527  level++)
18528  {
18529  for (unsigned ee = 0; ee < n_tree_nodes; ee++)
18530  {
18531  // Element exists at this level
18532  if (flat_packed_refinement_info_for_root[e][local_count] == 1)
18533  {
18534  local_count++;
18535 
18536  // Element should be refined
18537  if (flat_packed_refinement_info_for_root[e][local_count] == 1)
18538  {
18539  refinement_info_for_root_elements[e][level].push_back(2);
18540  local_count++;
18541  }
18542  // Element should not be refined
18543  else
18544  {
18545  refinement_info_for_root_elements[e][level].push_back(1);
18546  local_count++;
18547  }
18548  }
18549  // Element does not exist at this level
18550  else
18551  {
18552  refinement_info_for_root_elements[e][level].push_back(0);
18553  local_count++;
18554  }
18555  }
18556  }
18557 
18558 #ifdef PARANOID
18559  if (n_additional_data != local_count)
18560  {
18561  std::stringstream error_message;
18562  error_message << "Number of additional data: " << n_additional_data
18563  << " doesn't match that actually send: "
18564  << local_count << std::endl;
18565  throw OomphLibError(error_message.str(),
18566  OOMPH_CURRENT_FUNCTION,
18567  OOMPH_EXCEPTION_LOCATION);
18568  }
18569 #endif
18570  }
18571  // Element in question is not one of mine so prepare for sending
18572  //--------------------------------------------------------------
18573  else
18574  {
18575  // Make space
18576  unsigned current_size = data_for_proc[new_domain].size();
18577  unsigned n_additional_data =
18578  flat_packed_refinement_info_for_root[e].size();
18579  data_for_proc[new_domain].reserve(current_size + n_additional_data +
18580  2);
18581 
18582  // Keep counting
18583  count += n_additional_data + 2;
18584 
18585  // Add base element number
18586  data_for_proc[new_domain].push_back(e);
18587 
18588 #ifdef PARANOID
18589  // Add number of flat-packed instructions to follow
18590  data_for_proc[new_domain].push_back(n_additional_data);
18591 #endif
18592 
18593  // Add flat packed refinement data
18594  for (unsigned j = 0; j < n_additional_data; j++)
18595  {
18596  data_for_proc[new_domain].push_back(
18597  flat_packed_refinement_info_for_root[e][j]);
18598  }
18599  }
18600  }
18601  }
18602 
18603 
18604  // Now do the actual send/receive
18605  //-------------------------------
18606 
18607  // Storage for number of data to be sent to each processor
18608  Vector<int> send_n(n_proc, 0);
18609 
18610  // Storage for all values to be sent to all processors
18611  Vector<unsigned> send_data;
18612  send_data.reserve(count);
18613 
18614  // Start location within send_data for data to be sent to each processor
18615  Vector<int> send_displacement(n_proc, 0);
18616 
18617  // Loop over all processors
18618  for (int rank = 0; rank < n_proc; rank++)
18619  {
18620  // Set the offset for the current processor
18621  send_displacement[rank] = send_data.size();
18622 
18623  // Don't bother to do anything if the processor in the loop is the
18624  // current processor
18625  if (rank != my_rank)
18626  {
18627  // Record how many base elements are to be sent
18628  send_data.push_back(nbase_elements_for_proc[rank]);
18629 
18630  // Add data
18631  unsigned n_data = data_for_proc[rank].size();
18632  for (unsigned j = 0; j < n_data; j++)
18633  {
18634  send_data.push_back(data_for_proc[rank][j]);
18635  }
18636  }
18637 
18638  // Find the number of data added to the vector
18639  send_n[rank] = send_data.size() - send_displacement[rank];
18640  }
18641 
18642  // Storage for the number of data to be received from each processor
18643  Vector<int> receive_n(n_proc, 0);
18644 
18645  // Now send numbers of data to be sent between all processors
18646  MPI_Alltoall(&send_n[0],
18647  1,
18648  MPI_INT,
18649  &receive_n[0],
18650  1,
18651  MPI_INT,
18652  this->communicator_pt()->mpi_comm());
18653 
18654  // We now prepare the data to be received
18655  // by working out the displacements from the received data
18656  Vector<int> receive_displacement(n_proc, 0);
18657  int receive_data_count = 0;
18658  for (int rank = 0; rank < n_proc; ++rank)
18659  {
18660  // Displacement is number of data received so far
18661  receive_displacement[rank] = receive_data_count;
18662  receive_data_count += receive_n[rank];
18663  }
18664 
18665  // Now resize the receive buffer for all data from all processors
18666  // Make sure that it has a size of at least one
18667  if (receive_data_count == 0)
18668  {
18669  ++receive_data_count;
18670  }
18671  Vector<unsigned> receive_data(receive_data_count);
18672 
18673  // Make sure that the send buffer has size at least one
18674  // so that we don't get a segmentation fault
18675  if (send_data.size() == 0)
18676  {
18677  send_data.resize(1);
18678  }
18679 
18680  // Now send the data between all the processors
18681  MPI_Alltoallv(&send_data[0],
18682  &send_n[0],
18683  &send_displacement[0],
18684  MPI_UNSIGNED,
18685  &receive_data[0],
18686  &receive_n[0],
18687  &receive_displacement[0],
18688  MPI_UNSIGNED,
18689  this->communicator_pt()->mpi_comm());
18690 
18691 
18692  // Now use the received data to update
18693  //-----------------------------------
18694  for (int send_rank = 0; send_rank < n_proc; send_rank++)
18695  {
18696  // Don't bother to do anything for the processor corresponding to the
18697  // current processor or if no data were received from this processor
18698  if ((send_rank != my_rank) && (receive_n[send_rank] != 0))
18699  {
18700  // Counter for the data within the large array
18701  unsigned count = receive_displacement[send_rank];
18702 
18703  // Loop over base elements
18704  unsigned nbase_element = receive_data[count];
18705  count++;
18706  for (unsigned b = 0; b < nbase_element; b++)
18707  {
18708  // Get base element number
18709  unsigned base_element_number = receive_data[count];
18710  count++;
18711 
18712  // Record on which other procs/domains the refinement info for
18713  // this element is required because it's haloed.
18714  unsigned nhalo = halo_domains[base_element_number].size();
18715  halo_domain_of_haloed_base_element[base_element_number].resize(nhalo);
18716  for (unsigned j = 0; j < nhalo; j++)
18717  {
18718  halo_domain_of_haloed_base_element[base_element_number][j] =
18719  halo_domains[base_element_number][j];
18720  }
18721 
18722  // Provide storage for refinement pattern
18723  refinement_info_for_root_elements[base_element_number].resize(
18724  max_refinement_level_overall);
18725 
18726  // Get number of flat-packed instructions to follow
18727  // (only used for check)
18728 #ifdef PARANOID
18729  unsigned n_additional_data = receive_data[count];
18730  count++;
18731 
18732  // Counter for number of additional data (validation only)
18733  unsigned check_count = 0;
18734 #endif
18735 
18736  // Get number of tree nodes
18737  unsigned n_tree_nodes = receive_data[count];
18738  count++;
18739 
18740 #ifdef PARANOID
18741  check_count++;
18742 #endif
18743 
18744  // Loop over levels and number of nodes in tree
18745  for (unsigned level = 0; level < max_refinement_level_overall;
18746  level++)
18747  {
18748  for (unsigned e = 0; e < n_tree_nodes; e++)
18749  {
18750  // Element exists at this level
18751  if (receive_data[count] == 1)
18752  {
18753  count++;
18754 
18755 #ifdef PARANOID
18756  check_count++;
18757 #endif
18758 
18759  // Element should be refined
18760  if (receive_data[count] == 1)
18761  {
18762  refinement_info_for_root_elements[base_element_number][level]
18763  .push_back(2);
18764  count++;
18765 
18766 #ifdef PARANOID
18767  check_count++;
18768 #endif
18769  }
18770  // Element should not be refined
18771  else
18772  {
18773  refinement_info_for_root_elements[base_element_number][level]
18774  .push_back(1);
18775  count++;
18776 
18777 #ifdef PARANOID
18778  check_count++;
18779 #endif
18780  }
18781  }
18782  // Element does not exist at this level
18783  else
18784  {
18785  refinement_info_for_root_elements[base_element_number][level]
18786  .push_back(0);
18787  count++;
18788 
18789 #ifdef PARANOID
18790  check_count++;
18791 #endif
18792  }
18793  }
18794  }
18795 
18796 #ifdef PARANOID
18797  if (n_additional_data != check_count)
18798  {
18799  std::stringstream error_message;
18800  error_message << "Number of additional data: " << n_additional_data
18801  << " doesn't match that actually send: "
18802  << check_count << std::endl;
18803  throw OomphLibError(error_message.str(),
18804  OOMPH_CURRENT_FUNCTION,
18805  OOMPH_EXCEPTION_LOCATION);
18806  }
18807 #endif
18808  }
18809  }
18810  }
18811 
18812 
18813  // Now send the fully assembled refinement info to halo elements
18814  //---------------------------------------------------------------
18815  {
18816  // Accumulate data to be sent
18817  //---------------------------
18818 
18819  // Map to accumulate data to be sent to other procs
18820  // (map for sparsity)
18821  std::map<unsigned, Vector<unsigned>> data_for_proc;
18822 
18823  // Number of base elements to be sent to specified domain
18824  Vector<unsigned> nbase_elements_for_proc(n_proc, 0);
18825 
18826  // Loop over all haloed root elements and find out which
18827  // processors they have haloes on
18828  for (std::map<unsigned, Vector<unsigned>>::iterator it =
18829  halo_domain_of_haloed_base_element.begin();
18830  it != halo_domain_of_haloed_base_element.end();
18831  it++)
18832  {
18833  // Get base element number
18834  unsigned base_element_number = (*it).first;
18835 
18836  // Loop over target domains
18837  Vector<unsigned> domains = (*it).second;
18838  unsigned nd = domains.size();
18839  for (unsigned jd = 0; jd < nd; jd++)
18840  {
18841  // Actual number of domain
18842  unsigned d = domains[jd];
18843 
18844  // Keep counting number of base elemements for domain
18845  nbase_elements_for_proc[d]++;
18846 
18847  // Write base element number
18848  data_for_proc[d].push_back(base_element_number);
18849 
18850  // Write refinement info in flat-packed form
18851  for (unsigned level = 0; level < max_refinement_level_overall;
18852  level++)
18853  {
18854  // Number of entries at each level
18855  unsigned n =
18856  refinement_info_for_root_elements[base_element_number][level]
18857  .size();
18858  data_for_proc[d].push_back(n);
18859  for (unsigned j = 0; j < n; j++)
18860  {
18861  data_for_proc[d].push_back(
18862  refinement_info_for_root_elements[base_element_number][level]
18863  [j]);
18864  }
18865  }
18866  }
18867  }
18868 
18869 
18870  // Do the actual send
18871  //-------------------
18872 
18873  // Storage for number of data to be sent to each processor
18874  Vector<int> send_n(n_proc, 0);
18875 
18876  // Storage for all values to be sent to all processors
18877  Vector<unsigned> send_data;
18878  send_data.reserve(count);
18879 
18880  // Start location within send_data for data to be sent to each processor
18881  Vector<int> send_displacement(n_proc, 0);
18882 
18883  // Loop over all processors
18884  for (int rank = 0; rank < n_proc; rank++)
18885  {
18886  // Set the offset for the current processor
18887  send_displacement[rank] = send_data.size();
18888 
18889  // Don't bother to do anything if the processor in the loop is the
18890  // current processor
18891  if (rank != my_rank)
18892  {
18893  // Record how many base elements are to be sent
18894  send_data.push_back(nbase_elements_for_proc[rank]);
18895 
18896  // Add data
18897  unsigned n_data = data_for_proc[rank].size();
18898  for (unsigned j = 0; j < n_data; j++)
18899  {
18900  send_data.push_back(data_for_proc[rank][j]);
18901  }
18902  }
18903  // Find the number of data added to the vector
18904  send_n[rank] = send_data.size() - send_displacement[rank];
18905  }
18906 
18907  // Storage for the number of data to be received from each processor
18908  Vector<int> receive_n(n_proc, 0);
18909 
18910  // Now send numbers of data to be sent between all processors
18911  MPI_Alltoall(&send_n[0],
18912  1,
18913  MPI_INT,
18914  &receive_n[0],
18915  1,
18916  MPI_INT,
18917  this->communicator_pt()->mpi_comm());
18918 
18919  // We now prepare the data to be received
18920  // by working out the displacements from the received data
18921  Vector<int> receive_displacement(n_proc, 0);
18922  int receive_data_count = 0;
18923  for (int rank = 0; rank < n_proc; ++rank)
18924  {
18925  // Displacement is number of data received so far
18926  receive_displacement[rank] = receive_data_count;
18927  receive_data_count += receive_n[rank];
18928  }
18929 
18930  // Now resize the receive buffer for all data from all processors
18931  // Make sure that it has a size of at least one
18932  if (receive_data_count == 0)
18933  {
18934  ++receive_data_count;
18935  }
18936  Vector<unsigned> receive_data(receive_data_count);
18937 
18938  // Make sure that the send buffer has size at least one
18939  // so that we don't get a segmentation fault
18940  if (send_data.size() == 0)
18941  {
18942  send_data.resize(1);
18943  }
18944 
18945  // Now send the data between all the processors
18946  MPI_Alltoallv(&send_data[0],
18947  &send_n[0],
18948  &send_displacement[0],
18949  MPI_UNSIGNED,
18950  &receive_data[0],
18951  &receive_n[0],
18952  &receive_displacement[0],
18953  MPI_UNSIGNED,
18954  this->communicator_pt()->mpi_comm());
18955 
18956 
18957  // Now use the received data
18958  //------------------------
18959  for (int send_rank = 0; send_rank < n_proc; send_rank++)
18960  {
18961  // Don't bother to do anything for the processor corresponding to the
18962  // current processor or if no data were received from this processor
18963  if ((send_rank != my_rank) && (receive_n[send_rank] != 0))
18964  {
18965  // Counter for the data within the large array
18966  unsigned count = receive_displacement[send_rank];
18967 
18968  // Read number of base elements
18969  unsigned nbase_element = receive_data[count];
18970  count++;
18971 
18972  for (unsigned e = 0; e < nbase_element; e++)
18973  {
18974  // Read base element number
18975  unsigned base_element_number = receive_data[count];
18976  count++;
18977 
18978  // Provide storage for refinement pattern
18979  refinement_info_for_root_elements[base_element_number].resize(
18980  max_refinement_level_overall);
18981 
18982  // Read refinement info in flat-packed form
18983  for (unsigned level = 0; level < max_refinement_level_overall;
18984  level++)
18985  {
18986  // Read number of entries at each level
18987  unsigned n = receive_data[count];
18988  count++;
18989 
18990  // Read entries
18991  for (unsigned j = 0; j < n; j++)
18992  {
18993  refinement_info_for_root_elements[base_element_number][level]
18994  .push_back(receive_data[count]);
18995  count++;
18996  }
18997  }
18998  }
18999  }
19000  }
19001  }
19002  }
19003 
19004  //==========================================================================
19005  /// Load balance helper routine: Send data to other
19006  /// processors during load balancing.
19007  /// - send_n: Input, number of data to be sent to each processor
19008  /// - send_data: Input, storage for all values to be sent to all processors
19009  /// - send_displacement: Input, start location within send_data for data to
19010  /// be sent to each processor
19011  //==========================================================================
19013  Vector<int>& send_n,
19014  Vector<double>& send_data,
19015  Vector<int>& send_displacement)
19016  {
19017  // Communicator info
19018  OomphCommunicator* comm_pt = this->communicator_pt();
19019  const int n_proc = comm_pt->nproc();
19020 
19021  // Storage for the number of data to be received from each processor
19022  Vector<int> receive_n(n_proc, 0);
19023 
19024  // Now send numbers of data to be sent between all processors
19025  MPI_Alltoall(&send_n[0],
19026  1,
19027  MPI_INT,
19028  &receive_n[0],
19029  1,
19030  MPI_INT,
19031  this->communicator_pt()->mpi_comm());
19032 
19033  // We now prepare the data to be received
19034  // by working out the displacements from the received data
19035  Vector<int> receive_displacement(n_proc, 0);
19036  int receive_data_count = 0;
19037  for (int rank = 0; rank < n_proc; ++rank)
19038  {
19039  // Displacement is number of data received so far
19040  receive_displacement[rank] = receive_data_count;
19041  receive_data_count += receive_n[rank];
19042  }
19043 
19044  // Now resize the receive buffer for all data from all processors
19045  // Make sure that it has a size of at least one
19046  if (receive_data_count == 0)
19047  {
19048  ++receive_data_count;
19049  }
19050  Vector<double> receive_data(receive_data_count);
19051 
19052  // Make sure that the send buffer has size at least one
19053  // so that we don't get a segmentation fault
19054  if (send_data.size() == 0)
19055  {
19056  send_data.resize(1);
19057  }
19058 
19059  // Now send the data between all the processors
19060  MPI_Alltoallv(&send_data[0],
19061  &send_n[0],
19062  &send_displacement[0],
19063  MPI_DOUBLE,
19064  &receive_data[0],
19065  &receive_n[0],
19066  &receive_displacement[0],
19067  MPI_DOUBLE,
19068  this->communicator_pt()->mpi_comm());
19069 
19070  unsigned el_count = 0;
19071 
19072  // Only do each node once
19073  Vector<std::map<Node*, bool>> node_done(n_proc);
19074 
19075  // Now use the received data to update the halo nodes
19076  for (int send_rank = 0; send_rank < n_proc; send_rank++)
19077  {
19078  // Don't bother to do anything if no data were received from this
19079  // processor
19080  // NOTE: We do have to loop over our own processor number to process
19081  // the data locally.
19082  if (receive_n[send_rank] != 0)
19083  {
19084  // Counter for the data within the large array
19085  unsigned count = receive_displacement[send_rank];
19086 
19087  // How many batches are there for current rank
19088  unsigned nbatch = unsigned(receive_data[count]);
19089  count++;
19090 
19091  // Loop over batches (containing leaves associated with root elements)
19092  for (unsigned b = 0; b < nbatch; b++)
19093  {
19094  // How many elements were received for this batch?
19095  unsigned nel = unsigned(receive_data[count]);
19096  count++;
19097 
19098  // Get the unique base/root element number of this batch
19099  // in unrefined mesh
19100  unsigned base_el_no = unsigned(receive_data[count]);
19101  count++;
19102 
19103  // Get pointer to base/root element from reverse lookup scheme
19104  GeneralisedElement* root_el_pt = Base_mesh_element_pt[base_el_no];
19105 
19106  // Vector for pointers to associated elements in batch
19107  Vector<GeneralisedElement*> batch_el_pt;
19108 
19109  // Is it a refineable element?
19110  RefineableElement* ref_root_el_pt =
19111  dynamic_cast<RefineableElement*>(root_el_pt);
19112  if (ref_root_el_pt != 0)
19113  {
19114  // Get all leaves associated with this base/root element
19115  Vector<Tree*> all_leaf_nodes_pt;
19116  ref_root_el_pt->tree_pt()->stick_leaves_into_vector(
19117  all_leaf_nodes_pt);
19118 
19119  // How many leaves are there?
19120  unsigned n_leaf = all_leaf_nodes_pt.size();
19121 
19122 #ifdef PARANOID
19123  if (n_leaf != nel)
19124  {
19125  std::ostringstream error_message;
19126  error_message
19127  << "Number of leaves: " << n_leaf << " "
19128  << " doesn't match number of elements sent in batch: " << nel
19129  << "\n";
19130  throw OomphLibError(error_message.str(),
19131  OOMPH_CURRENT_FUNCTION,
19132  OOMPH_EXCEPTION_LOCATION);
19133  }
19134 #endif
19135 
19136  // Loop over batch of elements associated with this base/root
19137  // element
19138  batch_el_pt.resize(n_leaf);
19139  for (unsigned e = 0; e < n_leaf; e++)
19140  {
19141  batch_el_pt[e] = all_leaf_nodes_pt[e]->object_pt();
19142  }
19143  }
19144  // Not refineable -- the batch contains just the root element itself
19145  else
19146  {
19147 #ifdef PARANOID
19148  if (1 != nel)
19149  {
19150  std::ostringstream error_message;
19151  error_message
19152  << "Non-refineable root element should only be associated with"
19153  << " one element but nel=" << nel << "\n";
19154  throw OomphLibError(error_message.str(),
19155  OOMPH_CURRENT_FUNCTION,
19156  OOMPH_EXCEPTION_LOCATION);
19157  }
19158 #endif
19159  batch_el_pt.push_back(root_el_pt);
19160  }
19161 
19162  // Now loop over all elements in batch
19163  for (unsigned e = 0; e < nel; e++)
19164  {
19165  GeneralisedElement* el_pt = batch_el_pt[e];
19166  el_count++;
19167 
19168  // FE?
19169  FiniteElement* fe_pt = dynamic_cast<FiniteElement*>(el_pt);
19170  if (fe_pt != 0)
19171  {
19172  // Loop over nodes
19173  unsigned nnod = fe_pt->nnode();
19174  for (unsigned j = 0; j < nnod; j++)
19175  {
19176  Node* nod_pt = fe_pt->node_pt(j);
19177  if (!node_done[send_rank][nod_pt])
19178  {
19179  node_done[send_rank][nod_pt] = true;
19180 
19181 
19182  // Read number of values (as double) to allow for resizing
19183  // before read (req'd in case we store data that
19184  // got introduced by attaching FaceElements to bulk)
19185  unsigned nval = unsigned(receive_data[count]);
19186  count++;
19187 
19188 #ifdef PARANOID
19189  // Does the size match?
19190  if (nval < nod_pt->nvalue())
19191  {
19192  std::ostringstream error_message;
19193  error_message
19194  << "Node has more values, namely " << nod_pt->nvalue()
19195  << ", than we're about to receive, namely " << nval
19196  << ". Something's wrong!\n";
19197  throw OomphLibError(error_message.str(),
19198  OOMPH_CURRENT_FUNCTION,
19199  OOMPH_EXCEPTION_LOCATION);
19200  }
19201 #endif
19202 
19203 
19204 #ifdef PARANOID
19205  // Check if it's been sent as a boundary node
19206  unsigned is_boundary_node = unsigned(receive_data[count]);
19207  count++;
19208 #endif
19209 
19210  // Check if it's actually a boundary node
19211  BoundaryNodeBase* bnod_pt =
19212  dynamic_cast<BoundaryNodeBase*>(nod_pt);
19213  if (bnod_pt != 0)
19214  {
19215 #ifdef PARANOID
19216  // Check if local and received status are consistent
19217  if (is_boundary_node != 1)
19218  {
19219  std::ostringstream error_message;
19220  error_message << "Local node is boundary node but "
19221  "information sent is\n"
19222  << "for non-boundary node\n";
19223  throw OomphLibError(error_message.str(),
19224  OOMPH_CURRENT_FUNCTION,
19225  OOMPH_EXCEPTION_LOCATION);
19226  }
19227 #endif
19228 
19229  // Do we have entries in the map?
19230  unsigned n_entry = unsigned(receive_data[count]);
19231  count++;
19232  if (n_entry > 0)
19233  {
19234  // Create storage, if it doesn't already exist, for the
19235  // map that will contain the position of the first entry
19236  // of this face element's additional values,
19237  if (
19238  bnod_pt
19239  ->index_of_first_value_assigned_by_face_element_pt() ==
19240  0)
19241  {
19242  bnod_pt
19244  new std::map<unsigned, unsigned>;
19245  }
19246 
19247  // Get pointer to the map of indices associated with
19248  // additional values created by face elements
19249  std::map<unsigned, unsigned>* map_pt =
19250  bnod_pt
19252 
19253  // Loop over number of entries in map
19254  for (unsigned i = 0; i < n_entry; i++)
19255  {
19256  // Read out pairs...
19257  unsigned first = unsigned(receive_data[count]);
19258  count++;
19259  unsigned second = unsigned(receive_data[count]);
19260  count++;
19261 
19262  // ...and assign
19263  (*map_pt)[first] = second;
19264  }
19265  }
19266  }
19267 #ifdef PARANOID
19268  // Not a boundary node
19269  else
19270  {
19271  // Check if local and received status are consistent
19272  if (is_boundary_node != 0)
19273  {
19274  std::ostringstream error_message;
19275  error_message << "Local node is not a boundary node but "
19276  "information \n"
19277  << "sent is for boundary node.\n";
19278  throw OomphLibError(error_message.str(),
19279  OOMPH_CURRENT_FUNCTION,
19280  OOMPH_EXCEPTION_LOCATION);
19281  }
19282  }
19283 #endif
19284 
19285  // Do we have to resize? This can happen if node was
19286  // resized (due to a FaceElement that hasn't been attached
19287  // yet here) when the send data was written. If so make space
19288  // for the data here
19289  if (nval > nod_pt->nvalue())
19290  {
19291  nod_pt->resize(nval);
19292  }
19293 
19294  // Now read the actual values
19295  nod_pt->read_values_from_vector(receive_data, count);
19296  }
19297  }
19298  }
19299 
19300  // Now add internal data
19301  el_pt->read_internal_data_values_from_vector(receive_data, count);
19302  }
19303  }
19304  }
19305  }
19306 
19307  // Now that this is done, we need to synchronise dofs to get
19308  // the halo element and node values correct
19309  bool do_halos = true;
19310  bool do_external_halos = false;
19311  this->synchronise_dofs(do_halos, do_external_halos);
19312 
19313  // Now rebuild global mesh if required
19314  unsigned n_mesh = nsub_mesh();
19315  if (n_mesh != 0)
19316  {
19317  bool do_halos = false;
19318  bool do_external_halos = true;
19319  this->synchronise_dofs(do_halos, do_external_halos);
19321  }
19322  }
19323 
19324 
19325  //==========================================================================
19326  /// Load balance helper routine: Get data to be sent to other
19327  /// processors during load balancing and other information about
19328  /// re-distribution.
19329  /// - target_domain_for_local_non_halo_element: Input, generated by METIS.
19330  /// target_domain_for_local_non_halo_element[e] contains the number
19331  /// of the domain [0,1,...,nproc-1] to which non-halo element e on THE
19332  /// CURRENT PROCESSOR ONLY has been assigned. The order of the non-halo
19333  /// elements is the same as in the Problem's mesh, with the halo
19334  /// elements being skipped.
19335  /// - send_n: Output, number of data to be sent to each processor
19336  /// - send_data: Output, storage for all values to be sent to all processors
19337  /// - send_displacement: Output, start location within send_data for data to
19338  /// be sent to each processor
19339  /// - max_refinement_level_overall: Output, max. refinement level of any
19340  /// element
19341  //==========================================================================
19343  const Vector<unsigned>& target_domain_for_local_non_halo_element,
19344  Vector<int>& send_n,
19345  Vector<double>& send_data,
19346  Vector<int>& send_displacement,
19347  Vector<unsigned>& old_domain_for_base_element,
19348  Vector<unsigned>& new_domain_for_base_element,
19349  unsigned& max_refinement_level_overall)
19350  {
19351  // Communicator info
19352  OomphCommunicator* comm_pt = this->communicator_pt();
19353  const int n_proc = comm_pt->nproc();
19354  const int my_rank = this->communicator_pt()->my_rank();
19355 
19356  //------------------------------------------------------------------------
19357  // Overall strategy: Loop over all elements (in structured meshes),
19358  // identify their corresponding root elements and move all associated
19359  // leaves together, collecting the leaves in batches.
19360  // ------------------------------------------------------------------------
19361 
19362  // Map to store whether the root element has been visited yet
19363  std::map<RefineableElement*, bool> root_el_done;
19364 
19365 #ifdef PARANOID
19366 
19367  // Map for checking if all elements associated with same root
19368  // have the same target processor
19369  std::map<RefineableElement*, unsigned> target_plus_one_for_root;
19370 
19371 #endif
19372 
19373  // Storage for maximum refinement level
19374  unsigned max_refinement_level = 0;
19375 
19376  // Storage for (vector of) elements associated with target domain
19377  // (stored in map for sparsity): element_for_processor[d][e] is pointer
19378  // to e-th element that's supposed to move onto processor (domain) d.
19379  std::map<unsigned, Vector<GeneralisedElement*>> element_for_processor;
19380 
19381  // Storage for the number of elements in a specified batch of leaf
19382  // elements, all of which are associated with the same root/base element:
19383  // nelement_batch_for_processor[d][j] is the number of (leaf)
19384  // elements (all associated with the same root) to be moved together to
19385  // domain/processor d, in the j-th batch of elements.
19386  std::map<unsigned, Vector<unsigned>> nelement_batch_for_processor;
19387 
19388  // Storage for the unique number of the root element (in the unrefined
19389  // base mesh) whose leaves are moved together in a batch:
19390  // base_element_for_element_batch_for_processo[d][j] is the number of
19391  // unique number of the root element (in the unrefined
19392  // base mesh) of all leaf elements (associated with that root),
19393  // to be moved together to domain/processor d, in the j-th batch of
19394  // elements.
19395  std::map<unsigned, Vector<unsigned>>
19396  base_element_for_element_batch_for_processor;
19397 
19398  // Record old and new domains for non-halo root elements (will be
19399  // communicated globally). Initialise to -1 so we can use max
19400  // to extract the right one via MPI_Allreduce.
19401  // NOTE: We communicate these globally to facilitate distribution
19402  // of refinement pattern. While the data itself can be
19403  // sent point-to-point for non-halo elements,
19404  // mesh refinement information also needs to be sent for
19405  // halo elements which aren't known yet.
19406  unsigned n_base_element = Base_mesh_element_pt.size();
19407  Vector<int> old_domain_for_base_element_local(n_base_element, -1);
19408  Vector<int> new_domain_for_base_element_local(n_base_element, -1);
19409 
19410  // Loop over all non-halo elements on current processor and identify roots
19411  // -------------------------------------------------------------------
19412  // All leaf elements in associated tree (must!) get moved together
19413  //----------------------------------------------------------------
19414  unsigned count_non_halo_el = 0;
19415  // Get the number of submeshs, if there are no submeshes, then
19416  // increase the counter so that the loop below also work for the only
19417  // one mesh in the problem
19418  unsigned n_mesh = nsub_mesh();
19419  if (n_mesh == 0)
19420  {
19421  n_mesh = 1;
19422  }
19423  // We need to know if there are structure meshes (with elements) as
19424  // part of the problem in order to perform (or not) the proper
19425  // communications
19426  bool are_there_structured_meshes = false;
19427  // Go for the nonhalo elements only in the TreeBaseMeshes
19428  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
19429  {
19430  // Only work with structured meshes
19431  TriangleMeshBase* sub_mesh_pt =
19432  dynamic_cast<TriangleMeshBase*>(mesh_pt(i_mesh));
19433  if (!(sub_mesh_pt != 0))
19434  {
19435  const unsigned nele = mesh_pt(i_mesh)->nelement();
19436  if (nele > 0)
19437  {
19438  // Change the flag to indicate that there are structured meshes
19439  // (with elements, because we may have meshes with face
19440  // elements and therefore zero elements at this point)
19441  are_there_structured_meshes = true;
19442  }
19443 
19444  for (unsigned e = 0; e < nele; e++)
19445  {
19446  GeneralisedElement* el_pt = mesh_pt(i_mesh)->element_pt(e);
19447  if (!el_pt->is_halo())
19448  {
19449  // New non-halo: Where is this element supposed to go to?
19450  //-------------------------------------------------------
19451  unsigned target_domain =
19452  target_domain_for_local_non_halo_element[count_non_halo_el];
19453 
19454  // Bump up counter for non-halo elements
19455  count_non_halo_el++;
19456 
19457  // Is it a root element? (It is, trivially, if it's not refineable)
19458  //------------------------------------------------------------------
19459  RefineableElement* ref_el_pt =
19460  dynamic_cast<RefineableElement*>(el_pt);
19461  if (ref_el_pt == 0)
19462  {
19463  // Not refineable so add element itself
19464  element_for_processor[target_domain].push_back(el_pt);
19465 
19466  // Number of elements associated with this root/base
19467  // element (just the element itself)
19468  nelement_batch_for_processor[target_domain].push_back(1);
19469 
19470  // This is the unique base/root element number in unrefined mesh
19471  unsigned element_number_in_base_mesh =
19473 #ifdef PARANOID
19474  if (element_number_in_base_mesh == 0)
19475  {
19476  throw OomphLibError("Base_mesh_element_number_plus_one[...]=0",
19477  OOMPH_CURRENT_FUNCTION,
19478  OOMPH_EXCEPTION_LOCATION);
19479  }
19480 #endif
19481  element_number_in_base_mesh -= 1;
19482  base_element_for_element_batch_for_processor[target_domain]
19483  .push_back(element_number_in_base_mesh);
19484 
19485  /// Where do I come from, where do I go to?
19486  old_domain_for_base_element_local[element_number_in_base_mesh] =
19487  my_rank;
19488  new_domain_for_base_element_local[element_number_in_base_mesh] =
19489  target_domain;
19490  } // if (ref_el_pt==0)
19491  // It's not a root element so we package its leaves into a batch
19492  //--------------------------------------------------------------
19493  // of elements
19494  //------------
19495  else
19496  {
19497  // Get the root element
19498  RefineableElement* root_el_pt = ref_el_pt->root_element_pt();
19499 
19500  // Has this root been visited yet?
19501  if (!root_el_done[root_el_pt])
19502  {
19503  // Now we've done it
19504  root_el_done[root_el_pt] = true;
19505 
19506  // Unique number of root element in base mesh
19507  unsigned element_number_in_base_mesh =
19509 #ifdef PARANOID
19510  if (element_number_in_base_mesh == 0)
19511  {
19512  throw OomphLibError(
19513  "Base_mesh_element_number_plus_one[...]=0",
19514  OOMPH_CURRENT_FUNCTION,
19515  OOMPH_EXCEPTION_LOCATION);
19516  }
19517 #endif
19518  element_number_in_base_mesh -= 1;
19519 
19520  /// Where do I come from, where do I go to?
19521  old_domain_for_base_element_local[element_number_in_base_mesh] =
19522  my_rank;
19523  new_domain_for_base_element_local[element_number_in_base_mesh] =
19524  target_domain;
19525 
19526 #ifdef PARANOID
19527  // Store target domain associated with this root element
19528  // (offset by one) to allow checking that all elements
19529  // with the same root move to the same processor
19530  target_plus_one_for_root[root_el_pt] = target_domain + 1;
19531 #endif
19532 
19533  // Package all leaves into batch of elements
19534  Vector<Tree*> all_leaf_nodes_pt;
19535  root_el_pt->tree_pt()->stick_leaves_into_vector(
19536  all_leaf_nodes_pt);
19537 
19538  // Number of leaves
19539  unsigned n_leaf = all_leaf_nodes_pt.size();
19540 
19541  // Number of elements associated with this root/base element
19542  // (all the leaves)
19543  nelement_batch_for_processor[target_domain].push_back(n_leaf);
19544 
19545  // Store the unique base/root element number in unrefined mesh
19546  base_element_for_element_batch_for_processor[target_domain]
19547  .push_back(element_number_in_base_mesh);
19548 
19549  // Loop over leaves
19550  for (unsigned i_leaf = 0; i_leaf < n_leaf; i_leaf++)
19551  {
19552  // Add element object at leaf
19553  RefineableElement* leaf_el_pt =
19554  all_leaf_nodes_pt[i_leaf]->object_pt();
19555  element_for_processor[target_domain].push_back(leaf_el_pt);
19556 
19557  // Monitor/update maximum refinement level
19558  unsigned level = all_leaf_nodes_pt[i_leaf]->level();
19559  if (level > max_refinement_level)
19560  {
19561  max_refinement_level = level;
19562  }
19563  }
19564  }
19565 
19566 #ifdef PARANOID
19567  // Root element has already been visited
19568  else
19569  {
19570  // We don't have to do anything with this element since it's
19571  // already been processed earlier, but check that it's scheduled
19572  // to go onto the same processor as its root.
19573  if ((target_plus_one_for_root[root_el_pt] - 1) != target_domain)
19574  {
19575  std::ostringstream error_message;
19576  error_message
19577  << "All elements associated with same root must have "
19578  << "same target. during load balancing\n";
19579  throw OomphLibError(error_message.str(),
19580  OOMPH_CURRENT_FUNCTION,
19581  OOMPH_EXCEPTION_LOCATION);
19582  }
19583  }
19584 #endif
19585  } // else if (ref_el_pt==0)
19586  } // if (!ele_pt->is_halo())
19587  } // for (e < nele)
19588  } // if (!(sub_mesh_pt!=0))
19589  } // for (i_mesh < n_mesh)
19590 
19591 #ifdef PARANOID
19592  // Have we processed all target domains?
19593  if (target_domain_for_local_non_halo_element.size() != count_non_halo_el)
19594  {
19595  std::ostringstream error_message;
19596  error_message
19597  << "Have processed " << count_non_halo_el << " of "
19598  << target_domain_for_local_non_halo_element.size()
19599  << " target domains for local non-halo elelemts. \n "
19600  << "Very Odd -- we do (now) strip out the information for elements\n"
19601  << "that are removed in actions_before_distribute()...\n";
19602  throw OomphLibError(
19603  error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
19604  }
19605 #endif
19606 
19607  // Determine max. refinement level and origin/destination scheme
19608  // -------------------------------------------------------------
19609  // for all root/base elements
19610  // --------------------------
19611 
19612  // Allreduce to work out max max refinement level across all processors
19613  max_refinement_level_overall = 0;
19614 
19615  // Only perform this communications if necessary (it means if there
19616  // are structured meshes as part of the problem)
19617  if (are_there_structured_meshes)
19618  {
19619  MPI_Allreduce(&max_refinement_level,
19620  &max_refinement_level_overall,
19621  1,
19622  MPI_UNSIGNED,
19623  MPI_MAX,
19624  comm_pt->mpi_comm());
19625  } // if (are_there_structured_meshes)
19626 
19627  // Allreduce to tell everybody about the original and new domains
19628  // for root elements
19629  Vector<int> tmp_old_domain_for_base_element(n_base_element);
19630 
19631  // Only perform this communications if necessary (it means if there
19632  // are structured meshes as part of the problem)
19633  if (are_there_structured_meshes)
19634  {
19635  MPI_Allreduce(&old_domain_for_base_element_local[0],
19636  &tmp_old_domain_for_base_element[0],
19637  n_base_element,
19638  MPI_INT,
19639  MPI_MAX,
19640  comm_pt->mpi_comm());
19641  } // if (are_there_structured_meshes)
19642 
19643  Vector<int> tmp_new_domain_for_base_element(n_base_element);
19644  // Only perform this communications if necessary (it means if there
19645  // are structured meshes as part of the problem)
19646  if (are_there_structured_meshes)
19647  {
19648  MPI_Allreduce(&new_domain_for_base_element_local[0],
19649  &tmp_new_domain_for_base_element[0],
19650  n_base_element,
19651  MPI_INT,
19652  MPI_MAX,
19653  comm_pt->mpi_comm());
19654  } // if (are_there_structured_meshes)
19655 
19656  // Copy across (after optional sanity check)
19657  old_domain_for_base_element.resize(n_base_element);
19658  new_domain_for_base_element.resize(n_base_element);
19659  for (unsigned j = 0; j < n_base_element; j++)
19660  {
19661 #ifdef PARANOID
19662  if (tmp_old_domain_for_base_element[j] == -1)
19663  {
19664  std::ostringstream error_message;
19665  error_message << "Old domain for base element " << j << ": "
19666  << Base_mesh_element_pt[j]
19667  << "or its incarnation as refineable el: "
19668  << dynamic_cast<RefineableElement*>(
19670  << " which is of type "
19671  << typeid(*Base_mesh_element_pt[j]).name()
19672  << " does not\n"
19673  << "appear to have been assigned by any processor\n";
19674  throw OomphLibError(error_message.str(),
19675  OOMPH_CURRENT_FUNCTION,
19676  OOMPH_EXCEPTION_LOCATION);
19677  }
19678 #endif
19679  old_domain_for_base_element[j] = tmp_old_domain_for_base_element[j];
19680 #ifdef PARANOID
19681  if (tmp_new_domain_for_base_element[j] == -1)
19682  {
19683  std::ostringstream error_message;
19684  error_message << "New domain for base element " << j
19685  << "which is of type "
19686  << typeid(*Base_mesh_element_pt[j]).name()
19687  << " does not\n"
19688  << "appear to have been assigned by any processor\n";
19689  throw OomphLibError(error_message.str(),
19690  OOMPH_CURRENT_FUNCTION,
19691  OOMPH_EXCEPTION_LOCATION);
19692  }
19693 #endif
19694  new_domain_for_base_element[j] = tmp_new_domain_for_base_element[j];
19695  }
19696 
19697 
19698  // Loop over all processors and accumulate data to be sent
19699  //--------------------------------------------------------
19700  send_data.clear();
19701 
19702  // Only do each node once (per processor!)
19703  Vector<std::map<Node*, bool>> node_done(n_proc);
19704 
19705  // Loop over all processors. NOTE: We include current processor
19706  // since we have to refine local elements too -- store their data
19707  // in same data structure as the one used for off-processor elements.
19708  for (int rank = 0; rank < n_proc; rank++)
19709  {
19710  // Set the offset for the current processor
19711  send_displacement[rank] = send_data.size();
19712 
19713 #ifdef PARANOID
19714  // Check that total number of elements processed matches those
19715  // in individual batches
19716  unsigned total_nel = element_for_processor[rank].size();
19717 #endif
19718 
19719  // Counter for number of elements
19720  unsigned el_count = 0;
19721 
19722  // How many baches are there for current rank?
19723  unsigned nbatch = nelement_batch_for_processor[rank].size();
19724 
19725  // Add to vector of doubles to save on number of comms
19726  send_data.push_back(double(nbatch));
19727 
19728  // Loop over batches of elemnts associated with same root
19729  for (unsigned b = 0; b < nbatch; b++)
19730  {
19731  // How many elements are to be sent in this batch?
19732  unsigned nel = nelement_batch_for_processor[rank][b];
19733 
19734  // Get the unique number of the root element in unrefined mesh for
19735  // all the elements in this batch
19736  unsigned base_el_no =
19737  base_element_for_element_batch_for_processor[rank][b];
19738 
19739  // Add unsigneds to send data to minimise number of
19740  // communications
19741  send_data.push_back(double(nel));
19742  send_data.push_back(double(base_el_no));
19743 
19744  // Loop over batch of elements
19745  for (unsigned e = 0; e < nel; e++)
19746  {
19747  // Get element
19748  GeneralisedElement* el_pt = element_for_processor[rank][el_count];
19749 
19750  // FE?
19751  FiniteElement* fe_pt = dynamic_cast<FiniteElement*>(el_pt);
19752  if (fe_pt != 0)
19753  {
19754  // Loop over nodes
19755  unsigned nnod = fe_pt->nnode();
19756  for (unsigned j = 0; j < nnod; j++)
19757  {
19758  Node* nod_pt = fe_pt->node_pt(j);
19759 
19760  // Reconstruct the nodal values/position from the node's
19761  // possible hanging node representation to be on the safe side
19762  unsigned n_value = nod_pt->nvalue();
19763  unsigned nt = nod_pt->ntstorage();
19764  Vector<double> values(n_value);
19765  unsigned n_dim = nod_pt->ndim();
19766  Vector<double> position(n_dim);
19767 
19768  // Loop over all history values
19769  for (unsigned t = 0; t < nt; t++)
19770  {
19771  nod_pt->value(t, values);
19772  for (unsigned i = 0; i < n_value; i++)
19773  {
19774  nod_pt->set_value(t, i, values[i]);
19775  }
19776  nod_pt->position(t, position);
19777  for (unsigned i = 0; i < n_dim; i++)
19778  {
19779  nod_pt->x(t, i) = position[i];
19780  }
19781  }
19782 
19783 
19784  // Has the node already been done for current rank?
19785  if (!node_done[rank][nod_pt])
19786  {
19787  // Now it has been done
19788  node_done[rank][nod_pt] = true;
19789 
19790  // Store number of values (as double) to allow for resizing
19791  // before read (req'd in case we store data that
19792  // got introduced by attaching FaceElements to bulk)
19793  send_data.push_back(double(n_value));
19794 
19795  // Check if it's a boundary node
19796  BoundaryNodeBase* bnod_pt =
19797  dynamic_cast<BoundaryNodeBase*>(nod_pt);
19798 
19799  // Not a boundary node
19800  if (bnod_pt == 0)
19801  {
19802 #ifdef PARANOID
19803  // Record status for checking
19804  send_data.push_back(double(0));
19805 #endif
19806  }
19807  // Yes it's a boundary node
19808  else
19809  {
19810 #ifdef PARANOID
19811  // Record status for checking
19812  send_data.push_back(double(1));
19813 #endif
19814  // Get pointer to the map of indices associated with
19815  // additional values created by face elements
19816  std::map<unsigned, unsigned>* map_pt =
19818 
19819  // No additional values created
19820  if (map_pt == 0)
19821  {
19822  send_data.push_back(double(0));
19823  }
19824  // Created additional values
19825  else
19826  {
19827  // How many?
19828  send_data.push_back(double(map_pt->size()));
19829 
19830  // Loop over entries in map and add to send data
19831  for (std::map<unsigned, unsigned>::iterator p =
19832  map_pt->begin();
19833  p != map_pt->end();
19834  p++)
19835  {
19836  send_data.push_back(double((*p).first));
19837  send_data.push_back(double((*p).second));
19838  }
19839  }
19840  }
19841 
19842  // Add the actual values
19843  nod_pt->add_values_to_vector(send_data);
19844  }
19845  }
19846  }
19847 
19848  // Now add internal data
19849  el_pt->add_internal_data_values_to_vector(send_data);
19850 
19851  // Bump up counter in long vector of elements
19852  el_count++;
19853  }
19854  }
19855 
19856 
19857 #ifdef PARANOID
19858  // Check that total number of elements matches the total of those
19859  // in batches
19860  if (total_nel != el_count)
19861  {
19862  std::ostringstream error_message;
19863  error_message
19864  << "total_nel: " << total_nel << " "
19865  << " doesn't match total number of elements sent in batch: "
19866  << el_count << "\n";
19867  throw OomphLibError(error_message.str(),
19868  OOMPH_CURRENT_FUNCTION,
19869  OOMPH_EXCEPTION_LOCATION);
19870  }
19871 #endif
19872 
19873  // Find the number of data added to the vector
19874  send_n[rank] = send_data.size() - send_displacement[rank];
19875  }
19876  }
19877 
19878 
19879  //==========================================================================
19880  /// Get flat-packed refinement pattern for each root element in current
19881  /// mesh (labeled by unique number of root element in unrefined base mesh).
19882  /// The vector stored for each root element contains the following
19883  /// information:
19884  /// - First entry: Number of tree nodes (not just leaves!) in refinement
19885  /// tree emanating from this root [Zero if root element is not refineable]
19886  /// - Loop over all refinement levels
19887  /// - Loop over all tree nodes (not just leaves!)
19888  /// - If associated element exists when the mesh has been refined to
19889  /// this level (either because it has been refined to this level or
19890  /// because it's less refined): 1
19891  /// - If the element is to be refined: 1; else: 0
19892  /// - else (element doesn't exist when mesh is refined to this level
19893  /// (because it's more refined): 0
19894  /// .
19895  /// .
19896  /// .
19897  //==========================================================================
19899  const Vector<unsigned>& old_domain_for_base_element,
19900  const Vector<unsigned>& new_domain_for_base_element,
19901  const unsigned& max_refinement_level_overall,
19902  std::map<unsigned, Vector<unsigned>>& flat_packed_refinement_info_for_root)
19903  {
19904  // Map to store whether the root element has been visited yet
19905  std::map<RefineableElement*, bool> root_el_done;
19906 
19907  // Get the number of submeshs, if there are no submeshes, then
19908  // increase the counter so that the loop below also work for the only
19909  // one mesh in the problem
19910  unsigned n_mesh = nsub_mesh();
19911  if (n_mesh == 0)
19912  {
19913  n_mesh = 1;
19914  }
19915  // Go for the nonhalo elements only in the TreeBaseMeshes
19916  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
19917  {
19918  // Only work with structured
19919  TriangleMeshBase* sub_mesh_pt =
19920  dynamic_cast<TriangleMeshBase*>(mesh_pt(i_mesh));
19921  if (!(sub_mesh_pt != 0))
19922  {
19923  const unsigned nele_submesh = mesh_pt(i_mesh)->nelement();
19924  for (unsigned e = 0; e < nele_submesh; e++)
19925  {
19926  // Get pointer to element
19927  GeneralisedElement* el_pt = mesh_pt(i_mesh)->element_pt(e);
19928 
19929  // Ignore halos
19930  if (!el_pt->is_halo())
19931  {
19932  // Is it refineable? No!
19933  RefineableElement* ref_el_pt =
19934  dynamic_cast<RefineableElement*>(el_pt);
19935  if (ref_el_pt == 0)
19936  {
19937  // The element is not refineable - stick a zero in refinement_info
19938  // indicating that there are no tree nodes following
19939  unsigned e = Base_mesh_element_number_plus_one[el_pt];
19940 #ifdef PARANOID
19941  if (e == 0)
19942  {
19943  throw OomphLibError("Base_mesh_element_number_plus_one[...]=0",
19944  OOMPH_CURRENT_FUNCTION,
19945  OOMPH_EXCEPTION_LOCATION);
19946  }
19947 #endif
19948  e -= 1;
19949  flat_packed_refinement_info_for_root[e].push_back(0);
19950  }
19951  // Refineable
19952  else
19953  {
19954  // Get the root element
19955  RefineableElement* root_el_pt = ref_el_pt->root_element_pt();
19956 
19957  // Has this root been visited yet?
19958  if (!root_el_done[root_el_pt])
19959  {
19960  // Get unique number of root element in base mesh
19961  unsigned root_element_number =
19963 
19964 #ifdef PARANOID
19965  if (root_element_number == 0)
19966  {
19967  throw OomphLibError(
19968  "Base_mesh_element_number_plus_one[...]=0",
19969  OOMPH_CURRENT_FUNCTION,
19970  OOMPH_EXCEPTION_LOCATION);
19971  }
19972 #endif
19973  root_element_number -= 1;
19974 
19975  // Get all the nodes associated with this root element
19976  Vector<Tree*> all_tree_nodes_pt;
19978  all_tree_nodes_pt);
19979 
19980  // How many tree nodes are there?
19981  unsigned n_tree_nodes = all_tree_nodes_pt.size();
19982  flat_packed_refinement_info_for_root[root_element_number]
19983  .push_back(n_tree_nodes);
19984 
19985  // Loop over all levels
19986  for (unsigned current_level = 0;
19987  current_level < max_refinement_level_overall;
19988  current_level++)
19989  {
19990  // Loop over all tree nodes
19991  for (unsigned e = 0; e < n_tree_nodes; e++)
19992  {
19993  // What's the level of this tree node?
19994  unsigned level = all_tree_nodes_pt[e]->level();
19995 
19996  // Element exists at this refinement level of the mesh
19997  // if it's at this level or it's at a lower level and a leaf
19998  if ((level == current_level) ||
19999  ((level < current_level) &&
20000  (all_tree_nodes_pt[e]->is_leaf())))
20001  {
20002  flat_packed_refinement_info_for_root[root_element_number]
20003  .push_back(1);
20004 
20005  // If it's at this level, and not a leaf, then it will
20006  // need to be refined in the new mesh
20007  if ((level == current_level) &&
20008  (!all_tree_nodes_pt[e]->is_leaf()))
20009  {
20010  flat_packed_refinement_info_for_root
20011  [root_element_number]
20012  .push_back(1);
20013  }
20014  // Element exists at this level and is a leaf so it
20015  // doesn't have to be refined
20016  else
20017  {
20018  flat_packed_refinement_info_for_root
20019  [root_element_number]
20020  .push_back(0);
20021  }
20022  }
20023  // Element does not exist at this level so it doesn't have
20024  // to be refined
20025  else
20026  {
20027  flat_packed_refinement_info_for_root[root_element_number]
20028  .push_back(0);
20029  }
20030  }
20031  }
20032  // Now we've done it
20033  root_el_done[root_el_pt] = true;
20034  }
20035  }
20036 
20037  } // if (!el_pt->is_halo())
20038  } // for (e < nele_submesh)
20039  } // if (!(sub_mesh_pt!=0))
20040  } // for (i_mesh < n_mesh)
20041  }
20042 
20043  //==========================================================================
20044  /// Load balance helper routine: Function performs max_level_overall
20045  /// successive refinements of the problem's mesh(es) using the following
20046  /// procdure: Given ID of root element, root_element_id, and current
20047  /// refinement level, level, the e-th entry in
20048  /// refinement_info_for_root_elements[root_element_id][level][e] is equal
20049  /// to 2 if the e-th element (using the enumeration when the mesh has been
20050  /// refined to the level-th level) is to be refined during the next
20051  /// refinement; it's 1 if it's not to be refined.
20052  //==========================================================================
20054  Vector<Vector<Vector<unsigned>>>& refinement_info_for_root_elements,
20055  const unsigned& max_level_overall)
20056  {
20057  // Loop over sub meshes
20058  unsigned n_sub_mesh = nsub_mesh();
20059  unsigned max_mesh = std::max(n_sub_mesh, unsigned(1));
20060  for (unsigned i_mesh = 0; i_mesh < max_mesh; i_mesh++)
20061  {
20062  // Choose the right mesh
20063  Mesh* my_mesh_pt = 0;
20064  if (n_sub_mesh == 0)
20065  {
20066  my_mesh_pt = mesh_pt();
20067  }
20068  else
20069  {
20070  my_mesh_pt = mesh_pt(i_mesh);
20071  }
20072 
20073  // Number of elements on this processor -- currently all elements
20074  // are "base" elements since the mesh hasn't been refined.
20075  unsigned n_el_on_this_proc = my_mesh_pt->nelement();
20076 
20077  // Storage for actual refinement pattern:
20078  // to_be_refined_on_this_proc[level][e] contains the element number
20079  // of the e-th element that is to refined at the level-th refinement level
20080  Vector<Vector<unsigned>> to_be_refined_on_this_proc(max_level_overall);
20081 
20082  // Count, at each level, the total number of elements in the mesh
20083  // (we can accumulate this because we know that elements are
20084  // enumerated tree by tree).
20085  Vector<unsigned> el_count_on_this_proc(max_level_overall, 0);
20086 
20087  // Loop over levels where refinement is taking place
20088  for (unsigned level = 0; level < max_level_overall; level++)
20089  {
20090  // Loop over roots = unrefined elements on this processor in order.
20091  // Note that this loops over the trees in unique order
20092  for (unsigned e = 0; e < n_el_on_this_proc; e++)
20093  {
20094  // Get the (root) element
20095  FiniteElement* el_pt = my_mesh_pt->finite_element_pt(e);
20096 
20097  // What is its unique number in the base mesh
20098  unsigned root_el_no = Base_mesh_element_number_plus_one[el_pt];
20099 #ifdef PARANOID
20100  if (root_el_no == 0)
20101  {
20102  throw OomphLibError("Base_mesh_element_number_plus_one[...]=0",
20103  OOMPH_CURRENT_FUNCTION,
20104  OOMPH_EXCEPTION_LOCATION);
20105  }
20106 #endif
20107  root_el_no -= 1;
20108 
20109  // Number of refinements to be performed starting from current
20110  // root element
20111  unsigned n_refinements =
20112  refinement_info_for_root_elements[root_el_no].size();
20113 
20114  // Perform refinement?
20115  if (level < n_refinements)
20116  {
20117  // Loop over elements at this level
20118  unsigned n_el =
20119  refinement_info_for_root_elements[root_el_no][level].size();
20120  for (unsigned ee = 0; ee < n_el; ee++)
20121  {
20122  // Refinement code 2: Element is to be refined at this
20123  // level
20124  if (refinement_info_for_root_elements[root_el_no][level][ee] == 2)
20125  {
20126  to_be_refined_on_this_proc[level].push_back(
20127  el_count_on_this_proc[level]);
20128  el_count_on_this_proc[level]++;
20129  }
20130  // Refinement code 1: Element should not be refined at this
20131  // level -- keep going
20132  else if (refinement_info_for_root_elements[root_el_no][level]
20133  [ee] == 1)
20134  {
20135  el_count_on_this_proc[level]++;
20136  }
20137  }
20138  }
20139 
20140  } // end of loop over elements on proc; all of which should be root
20141  }
20142 
20143  // Now do the actual refinement
20144  TreeBasedRefineableMeshBase* ref_mesh_pt =
20145  dynamic_cast<TreeBasedRefineableMeshBase*>(my_mesh_pt);
20146  if (ref_mesh_pt != 0)
20147  {
20148  ref_mesh_pt->refine_base_mesh(to_be_refined_on_this_proc);
20149  }
20150  }
20151 
20152  // Rebuild global mesh after refinement
20153  if (n_sub_mesh != 0)
20154  {
20155  // Rebuild the global mesh
20157  }
20158  }
20159 
20160 
20161  //====================================================================
20162  /// Helper function to re-setup the Base_mesh enumeration
20163  /// (used during load balancing) after pruning.
20164  //====================================================================
20166  {
20167  // Storage for number of processors and current processor
20168  int n_proc = this->communicator_pt()->nproc();
20169  int my_rank = this->communicator_pt()->my_rank();
20170 
20171  // Loop over sub meshes
20172  unsigned n_sub_mesh = nsub_mesh();
20173  unsigned max_mesh = std::max(n_sub_mesh, unsigned(1));
20174  for (unsigned i_mesh = 0; i_mesh < max_mesh; i_mesh++)
20175  {
20176  // Choose the right mesh
20177  Mesh* my_mesh_pt = 0;
20178  if (n_sub_mesh == 0)
20179  {
20180  my_mesh_pt = mesh_pt();
20181  }
20182  else
20183  {
20184  my_mesh_pt = mesh_pt(i_mesh);
20185  }
20186 
20187  // Only work with structured meshes
20188  TriangleMeshBase* sub_mesh_pt =
20189  dynamic_cast<TriangleMeshBase*>(my_mesh_pt);
20190  if (!(sub_mesh_pt != 0))
20191  {
20192  // Storage for number of data to be sent to each processor
20193  Vector<int> send_n(n_proc, 0);
20194 
20195  // Storage for all values to be sent to all processors
20196  Vector<unsigned> send_data;
20197 
20198  // Start location within send_data for data to be sent to each processor
20199  Vector<int> send_displacement(n_proc, 0);
20200 
20201  // Loop over all processors
20202  for (int rank = 0; rank < n_proc; rank++)
20203  {
20204  // Set the offset for the current processor
20205  send_displacement[rank] = send_data.size();
20206 
20207  // Don't bother to do anything if the processor in the loop is the
20208  // current processor
20209  if (rank != my_rank)
20210  {
20211  // Get root haloed elements with that processor
20212  Vector<GeneralisedElement*> root_haloed_elements_pt =
20213  my_mesh_pt->root_haloed_element_pt(rank);
20214  unsigned nel = root_haloed_elements_pt.size();
20215 
20216  // Store element numbers for send
20217  for (unsigned e = 0; e < nel; e++)
20218  {
20219  GeneralisedElement* el_pt = root_haloed_elements_pt[e];
20220  send_data.push_back(Base_mesh_element_number_plus_one[el_pt]);
20221  }
20222  }
20223 
20224  // Find the number of data added to the vector
20225  send_n[rank] = send_data.size() - send_displacement[rank];
20226  }
20227 
20228  // Storage for the number of data to be received from each processor
20229  Vector<int> receive_n(n_proc, 0);
20230 
20231  // Now send numbers of data to be sent between all processors
20232  MPI_Alltoall(&send_n[0],
20233  1,
20234  MPI_INT,
20235  &receive_n[0],
20236  1,
20237  MPI_INT,
20238  this->communicator_pt()->mpi_comm());
20239 
20240  // We now prepare the data to be received
20241  // by working out the displacements from the received data
20242  Vector<int> receive_displacement(n_proc, 0);
20243  int receive_data_count = 0;
20244  for (int rank = 0; rank < n_proc; ++rank)
20245  {
20246  // Displacement is number of data received so far
20247  receive_displacement[rank] = receive_data_count;
20248  receive_data_count += receive_n[rank];
20249  }
20250 
20251  // Now resize the receive buffer for all data from all processors
20252  // Make sure that it has a size of at least one
20253  if (receive_data_count == 0)
20254  {
20255  ++receive_data_count;
20256  }
20257  Vector<unsigned> receive_data(receive_data_count);
20258 
20259  // Make sure that the send buffer has size at least one
20260  // so that we don't get a segmentation fault
20261  if (send_data.size() == 0)
20262  {
20263  send_data.resize(1);
20264  }
20265 
20266  // Now send the data between all the processors
20267  MPI_Alltoallv(&send_data[0],
20268  &send_n[0],
20269  &send_displacement[0],
20270  MPI_UNSIGNED,
20271  &receive_data[0],
20272  &receive_n[0],
20273  &receive_displacement[0],
20274  MPI_UNSIGNED,
20275  this->communicator_pt()->mpi_comm());
20276 
20277  // Now use the received data to update the halo element numbers in
20278  // base mesh
20279  for (int send_rank = 0; send_rank < n_proc; send_rank++)
20280  {
20281  // Don't bother to do anything for the processor corresponding to the
20282  // current processor or if no data were received from this processor
20283  if ((send_rank != my_rank) && (receive_n[send_rank] != 0))
20284  {
20285  // Counter for the data within the large array
20286  unsigned count = receive_displacement[send_rank];
20287 
20288  // Get root halo elements with that processor
20289  Vector<GeneralisedElement*> root_halo_elements_pt =
20290  my_mesh_pt->root_halo_element_pt(send_rank);
20291  unsigned nel = root_halo_elements_pt.size();
20292 
20293  // Read in element numbers
20294  for (unsigned e = 0; e < nel; e++)
20295  {
20296  GeneralisedElement* el_pt = root_halo_elements_pt[e];
20297  unsigned el_number_plus_one = receive_data[count++];
20298  Base_mesh_element_number_plus_one[el_pt] = el_number_plus_one;
20299  Base_mesh_element_pt[el_number_plus_one - 1] = el_pt;
20300  }
20301  }
20302 
20303  } // End of data is received
20304 
20305  } // if (!(sub_mesh_pt!=0))
20306 
20307  } // for (i_mesh<max_mesh)
20308  }
20309 
20310 #endif
20311 
20312  /// Instantiation of public flag to allow suppression of warning
20313  /// messages re reading in unstructured meshes during restart.
20315  false;
20316 
20317 
20318 } // namespace oomph
e
Definition: cfortran.h:571
cstr elem_len * i
Definition: cfortran.h:603
char t
Definition: cfortran.h:568
Function-type-object to perform absolute comparison of objects. Apparently this inlines better.
A class that is used to define the functions used to assemble the elemental contributions to the resi...
virtual unsigned ndof(GeneralisedElement *const &elem_pt)
Return the number of degrees of freedom in the element elem_pt.
virtual void synchronise()
Function that is used to perform any synchronisation required during the solution.
virtual int bifurcation_type() const
Return an unsigned integer to indicate whether the handler is a bifurcation tracking handler....
virtual void get_hessian_vector_products(GeneralisedElement *const &elem_pt, Vector< double > const &Y, DenseMatrix< double > const &C, DenseMatrix< double > &product)
Calculate the product of the Hessian (derivative of Jacobian with respect to all variables) an eigenv...
virtual double * bifurcation_parameter_pt() const
Return a pointer to the bifurcation parameter in bifurcation tracking problems.
virtual void get_all_vectors_and_matrices(GeneralisedElement *const &elem_pt, Vector< Vector< double >> &vec, Vector< DenseMatrix< double >> &matrix)
Calculate all desired vectors and matrices provided by the element elem_pt.
virtual void get_eigenfunction(Vector< DoubleVector > &eigenfunction)
Return the eigenfunction(s) associated with the bifurcation that has been detected in bifurcation tra...
virtual void get_residuals(GeneralisedElement *const &elem_pt, Vector< double > &residuals)
Return the contribution to the residuals of the element elem_pt.
virtual unsigned long eqn_number(GeneralisedElement *const &elem_pt, const unsigned &ieqn_local)
Return the global equation number of the local unknown ieqn_local in elem_pt.
virtual void get_jacobian(GeneralisedElement *const &elem_pt, Vector< double > &residuals, DenseMatrix< double > &jacobian)
Calculate the elemental Jacobian matrix "d equation / d variable" for elem_pt.
A custom linear solver class that is used to solve a block-factorised version of the Hopf bifurcation...
A class that contains the information required by Nodes that are located on Mesh boundaries....
Definition: nodes.h:1996
std::map< unsigned, unsigned > *& index_of_first_value_assigned_by_face_element_pt()
Return pointer to the map giving the index of the first face element value.
Definition: nodes.h:2046
//////////////////////////////////////////////////////////////// ////////////////////////////////////...
Definition: matrices.h:2791
void build_without_copy(T *value, int *row_index, int *column_start, const unsigned long &nnz, const unsigned long &n, const unsigned long &m)
Function to build matrix from pointers to arrays which hold the column starts, row indices and non-ze...
Definition: matrices.h:3199
A class for compressed row matrices. This is a distributable object.
Definition: matrices.h:888
void redistribute(const LinearAlgebraDistribution *const &dist_pt)
The contents of the matrix are redistributed to match the new distribution. In a non-MPI build this m...
Definition: matrices.cc:2575
void build_without_copy(const unsigned &ncol, const unsigned &nnz, double *value, int *column_index, int *row_start)
keeps the existing distribution and just matrix that is stored without copying the matrix data
Definition: matrices.cc:1710
unsigned long nrow() const
Return the number of rows of the matrix.
Definition: matrices.h:1002
void build(const LinearAlgebraDistribution *distribution_pt, const unsigned &ncol, const Vector< double > &value, const Vector< int > &column_index, const Vector< int > &row_start)
build method: vector of values, vector of column indices, vector of row starts and number of rows and...
Definition: matrices.cc:1672
void set_consistent_pinned_values(Data *const &data_pt)
Set consistent values of the derivatives and current value when the data is pinned....
A Base class for DGElements.
Definition: dg_elements.h:161
void disable_mass_matrix_reuse()
Function that disables the reuse of the mass matrix.
Definition: dg_elements.h:240
void enable_mass_matrix_reuse()
Function that allows the reuse of the mass matrix.
Definition: dg_elements.h:233
virtual void get_inverse_mass_matrix_times_residuals(Vector< double > &minv_res)
Function that returns the current value of the residuals multiplied by the inverse mass matrix (virtu...
Definition: dg_elements.cc:614
A class that represents a collection of data; each Data object may contain many different individual ...
Definition: nodes.h:86
long & eqn_number(const unsigned &i)
Return the equation number of the i-th stored variable.
Definition: nodes.h:367
virtual void add_eqn_numbers_to_vector(Vector< long > &vector_of_eqn_numbers)
Add all equation numbers to the vector in the internal storage order.
Definition: nodes.cc:1236
double * value_pt(const unsigned &i) const
Return the pointer to the i-the stored value. Typically this is required when direct access to the st...
Definition: nodes.h:324
void copy(Data *orig_data_pt)
Copy Data values from specified Data object.
Definition: nodes.cc:601
void set_value(const unsigned &i, const double &value_)
Set the i-th stored data value to specified value. The only reason that we require an explicit set fu...
Definition: nodes.h:271
unsigned nvalue() const
Return number of values stored in data object (incl pinned ones).
Definition: nodes.h:483
double value(const unsigned &i) const
Return i-th stored value. This function is not virtual so that it can be inlined. This means that if ...
Definition: nodes.h:293
unsigned ntstorage() const
Return total number of doubles stored per value to record time history of each value (one for steady ...
Definition: nodes.cc:879
virtual void read_eqn_numbers_from_vector(const Vector< long > &vector_of_eqn_numbers, unsigned &index)
Read all equation numbers from the vector starting from index. On return the index will be set to the...
Definition: nodes.cc:1274
bool is_pinned(const unsigned &i) const
Test whether the i-th variable is pinned (1: true; 0: false).
Definition: nodes.h:417
Class of matrices containing doubles, and stored as a DenseMatrix<double>, but with solving functiona...
Definition: matrices.h:1271
void initialise(const T &val)
Initialize all values in the matrix to val.
Definition: matrices.h:514
void resize(const unsigned long &n)
Resize to a square nxn matrix; any values already present will be transfered.
Definition: matrices.h:498
LinearAlgebraDistribution * distribution_pt() const
access to the LinearAlgebraDistribution
unsigned nrow() const
access function to the number of global rows.
bool distribution_built() const
if the communicator_pt is null then the distribution is not setup then false is returned,...
unsigned nrow_local() const
access function for the num of local rows on this processor.
Information for documentation of results: Directory and file number to enable output in the form RESL...
bool is_doc_enabled() const
Are we documenting?
void disable_doc()
Disable documentation.
std::string & label()
String used (e.g.) for labeling output files.
std::string directory() const
Output directory.
unsigned & number()
Number used (e.g.) for labeling output files.
A class that stores the halo/haloed entries required when using a DoubleVectorWithHaloEntries....
void setup_halo_dofs(const std::map< unsigned, double * > &halo_data_pt, Vector< double * > &halo_dof_pt)
Function that sets up a vector of pointers to halo data, index using the scheme in Local_index.
===================================================================== An extension of DoubleVector th...
void build_halo_scheme(DoubleVectorHaloScheme *const &halo_scheme_pt)
Construct the halo scheme and storage for the halo data.
void sum_all_halo_and_haloed_values()
Sum all the data, store in the master (haloed) data and then synchronise.
double & global_value(const unsigned &i)
Direct access to global entry.
A vector in the mathematical sense, initially developed for linear algebra type applications....
Definition: double_vector.h:58
void initialise(const double &v)
initialise the whole vector with value v
void set_external_values(const LinearAlgebraDistribution *const &dist_pt, double *external_values, bool delete_external_values)
Allows are external data to be used by this vector. WARNING: The size of the external data must corre...
double max() const
returns the maximum coefficient
void build(const DoubleVector &old_vector)
Just copys the argument DoubleVector.
double * values_pt()
access function to the underlying values
void redistribute(const LinearAlgebraDistribution *const &dist_pt)
The contents of the vector are redistributed to match the new distribution. In a non-MPI rebuild this...
bool built() const
double dot(const DoubleVector &vec) const
compute the dot product of this vector with the vector vec.
void clear()
wipes the DoubleVector
A class that is used to define the functions used to assemble the elemental contributions to the mass...
virtual void solve_eigenproblem(Problem *const &problem_pt, const int &n_eval, Vector< std::complex< double >> &eigenvalue, Vector< DoubleVector > &eigenvector_real, Vector< DoubleVector > &eigenvector_imag, const bool &do_adjoint_problem=false)
Solve the real eigenproblem that is assembled by elements in a mesh in a Problem object....
Definition: eigen_solver.h:89
Base class for spatial error estimators.
void get_element_errors(Mesh *&mesh_pt, Vector< double > &elemental_error)
Compute the elemental error-measures for a given mesh and store them in a vector.
A class that is used to define the functions used to assemble and invert the mass matrix when taking ...
A Base class for explicit timesteppers.
virtual void timestep(ExplicitTimeSteppableObject *const &object_pt, const double &dt)=0
Pure virtual function that is used to advance time in the object.
FaceElements are elements that coincide with the faces of higher-dimensional "bulk" elements....
Definition: elements.h:4342
int & face_index()
Index of the face (a number that uniquely identifies the face in the element)
Definition: elements.h:4630
FiniteElement *& bulk_element_pt()
Pointer to higher-dimensional "bulk" element.
Definition: elements.h:4739
A general Finite Element class.
Definition: elements.h:1317
Node *& node_pt(const unsigned &n)
Return a pointer to the local node n.
Definition: elements.h:2179
virtual void output(std::ostream &outfile)
Output the element data — typically the values at the nodes in a format suitable for post-processing.
Definition: elements.h:3054
unsigned nnode() const
Return the number of nodes.
Definition: elements.h:2214
A Generalised Element class.
Definition: elements.h:73
bool is_halo() const
Is this element a halo?
Definition: elements.h:1167
void read_internal_eqn_numbers_from_vector(const Vector< long > &vector_of_eqn_numbers, unsigned &index)
Read all equation numbers associated with internal data from the vector starting from index....
Definition: elements.cc:675
unsigned ndof() const
Return the number of equations/dofs in the element.
Definition: elements.h:839
unsigned long eqn_number(const unsigned &ieqn_local) const
Return the global equation number corresponding to the ieqn_local-th local equation number.
Definition: elements.h:708
void add_internal_data_values_to_vector(Vector< double > &vector_of_values)
Add all internal data and time history values to the vector in the internal storage order.
Definition: elements.cc:633
Data *& internal_data_pt(const unsigned &i)
Return a pointer to i-th internal data object.
Definition: elements.h:622
unsigned ninternal_data() const
Return the number of internal data objects.
Definition: elements.h:827
void add_internal_eqn_numbers_to_vector(Vector< long > &vector_of_eqn_numbers)
Add all equation numbers associated with internal data to the vector in the internal storage order.
Definition: elements.cc:660
virtual void complete_setup_of_dependencies()
Complete the setup of any additional dependencies that the element may have. Empty virtual function t...
Definition: elements.h:978
void read_internal_data_values_from_vector(const Vector< double > &vector_of_values, unsigned &index)
Read all internal data and time history values from the vector starting from index....
Definition: elements.cc:647
Class that contains data for hanging nodes.
Definition: nodes.h:742
double const & master_weight(const unsigned &i) const
Return weight for dofs on i-th master node.
Definition: nodes.h:808
Node *const & master_node_pt(const unsigned &i) const
Return a pointer to the i-th master node.
Definition: nodes.h:791
unsigned nmaster() const
Return the number of master nodes.
Definition: nodes.h:785
void set_master_node_pt(const unsigned &i, Node *const &master_node_pt, const double &weight)
Set the pointer to the i-th master node and its weight.
Definition: nodes.cc:1474
A class that is used to assemble the augmented system that defines a Hopf bifurcation....
//////////////////////////////////////////////////////////////////// ////////////////////////////////...
Definition: elements.h:5270
Class for the LAPACK QZ eigensolver.
Definition: eigen_solver.h:158
Describes the distribution of a distributable linear algebra type object. Typically this is a contain...
bool distributed() const
access function to the distributed - indicates whether the distribution is serial or distributed
unsigned first_row() const
access function for the first row on this processor. If not distributed then this is just zero.
OomphCommunicator * communicator_pt() const
const access to the communicator pointer
void build(const OomphCommunicator *const comm_pt, const unsigned &first_row, const unsigned &nrow_local, const unsigned &nrow=0)
Sets the distribution. Takes first_row, nrow_local and nrow as arguments. If nrow is not provided or ...
unsigned nrow() const
access function to the number of global rows.
unsigned nrow_local() const
access function for the num of local rows on this processor. If no MPI then Nrow is returned.
unsigned rank_of_global_row(const unsigned i) const
return the processor rank of the global row number i
virtual void solve(Problem *const &problem_pt, DoubleVector &result)=0
Solver: Takes pointer to problem and returns the results vector which contains the solution of the li...
virtual void enable_resolve()
Enable resolve (i.e. store matrix and/or LU decomposition, say) Virtual so it can be overloaded to pe...
virtual void enable_computation_of_gradient()
function to enable the computation of the gradient required for the globally convergent Newton method
virtual void resolve(const DoubleVector &rhs, DoubleVector &result)
Resolve the system defined by the last assembled jacobian and the rhs vector. Solution is returned in...
void get_gradient(DoubleVector &gradient)
function to access the gradient, provided it has been computed
void reset_gradient()
function to reset the size of the gradient before each Newton solve
virtual void disable_resolve()
Disable resolve (i.e. store matrix and/or LU decomposition, say) This function simply resets an inter...
bool is_resolve_enabled() const
Boolean flag indicating if resolves are enabled.
static bool mpi_has_been_initialised()
return true if MPI has been initialised
static OomphCommunicator * communicator_pt()
access to global communicator. This is the oomph-lib equivalent of MPI_COMM_WORLD
A general mesh class.
Definition: mesh.h:67
void set_external_halo_node_pt(const unsigned &p, const Vector< Node * > &external_halo_node_pt)
Set vector of external halo node in this Mesh whose non-halo external counterpart is held on processo...
Definition: mesh.h:2398
Node * halo_node_pt(const unsigned &p, const unsigned &j)
Access fct to the j-th halo node in this Mesh whose non-halo counterpart is held on processor p.
Definition: mesh.h:1914
bool does_pointer_correspond_to_mesh_data(double *const &parameter_pt)
Does the double pointer correspond to any mesh data.
Definition: mesh.cc:2471
void remove_boundary_node(const unsigned &b, Node *const &node_pt)
Remove a node from the boundary b.
Definition: mesh.cc:221
bool is_mesh_distributed() const
Boolean to indicate if Mesh has been distributed.
Definition: mesh.h:1588
unsigned nexternal_haloed_element()
Total number of external haloed elements in this Mesh.
Definition: mesh.h:2267
Node * haloed_node_pt(const unsigned &p, const unsigned &j)
Access fct to the j-th haloed node in this Mesh whose halo counterpart is held on processor p.
Definition: mesh.h:2014
unsigned nexternal_halo_node()
Total number of external halo nodes in this Mesh.
Definition: mesh.h:2308
void flush_element_and_node_storage()
Flush storage for elements and nodes by emptying the vectors that store the pointers to them....
Definition: mesh.h:407
Node *& external_halo_node_pt(const unsigned &p, const unsigned &j)
Access fct to the j-th external halo node in this Mesh whose non-halo external counterpart is held on...
Definition: mesh.h:2377
FiniteElement * finite_element_pt(const unsigned &e) const
Upcast (downcast?) to FiniteElement (needed to access FiniteElement member functions).
Definition: mesh.h:473
void check_halo_schemes(DocInfo &doc_info, double &max_permitted_error_for_halo_check)
Check halo and shared schemes on the mesh.
Definition: mesh.cc:6881
virtual void set_mesh_level_time_stepper(TimeStepper *const &time_stepper_pt, const bool &preserve_existing_data)
Function that can be used to set any additional timestepper data stored at the Mesh (as opposed to no...
Definition: mesh.cc:2402
Vector< GeneralisedElement * > haloed_element_pt(const unsigned &p)
Return vector of haloed elements in this Mesh whose haloing counterpart is held on processor p.
Definition: mesh.h:1779
void describe_local_dofs(std::ostream &out, const std::string &current_string) const
Function to describe the local dofs of the elements. The ostream specifies the output stream to which...
Definition: mesh.cc:746
GeneralisedElement *& external_halo_element_pt(const unsigned &p, const unsigned &e)
Access fct to the e-th external halo element in this Mesh whose non-halo counterpart is held on proce...
Definition: mesh.h:2251
void set_nodal_and_elemental_time_stepper(TimeStepper *const &time_stepper_pt, const bool &preserve_existing_data)
Set the timestepper associated with all nodal and elemental data stored in the mesh.
Definition: mesh.h:1032
void set_external_haloed_node_pt(const unsigned &p, const Vector< Node * > &external_haloed_node_pt)
Set vector of external haloed node in this Mesh whose halo external counterpart is held on processor ...
Definition: mesh.h:2466
Node *& external_haloed_node_pt(const unsigned &p, const unsigned &j)
Access fct to the j-th external haloed node in this Mesh whose halo external counterpart is held on p...
Definition: mesh.h:2441
void shift_time_values()
Shift time-dependent data along for next timestep: Deal with nodal Data/positions and the element's i...
Definition: mesh.cc:2326
void prune_halo_elements_and_nodes(Vector< GeneralisedElement * > &deleted_element_pt, const bool &report_stats=false)
(Irreversibly) prune halo(ed) elements and nodes, usually after another round of refinement,...
Definition: mesh.h:1667
void calculate_predictions()
Calculate predictions for all Data and positions associated with the mesh, usually used in adaptive t...
Definition: mesh.cc:2366
void describe_dofs(std::ostream &out, const std::string &current_string) const
Function to describe the dofs of the Mesh. The ostream specifies the output stream to which the descr...
Definition: mesh.cc:711
GeneralisedElement *& element_pt(const unsigned long &e)
Return pointer to element e.
Definition: mesh.h:448
unsigned long nnode() const
Return number of nodes in the mesh.
Definition: mesh.h:596
void delete_all_external_storage()
Wipe the storage for all externally-based elements.
Definition: mesh.cc:9190
unsigned nnon_halo_element()
Total number of non-halo elements in this mesh (Costly call computes result on the fly)
Definition: mesh.h:1817
void get_all_halo_data(std::map< unsigned, double * > &map_of_halo_data)
Get all the halo data stored in the mesh and add pointers to the data to the map, indexed by global e...
Definition: mesh.cc:4749
unsigned nhalo_node()
Total number of halo nodes in this Mesh.
Definition: mesh.h:1878
void set_communicator_pt(OomphCommunicator *comm_pt)
Function to set communicator (mesh is assumed to be distributed if the communicator pointer is non-nu...
Definition: mesh.h:1615
void output(std::ostream &outfile)
Output for all elements.
Definition: mesh.cc:2027
void assign_initial_values_impulsive()
Assign initial values for an impulsive start.
Definition: mesh.cc:2288
Node *& node_pt(const unsigned long &n)
Return pointer to global node n.
Definition: mesh.h:436
void assign_local_eqn_numbers(const bool &store_local_dof_pt)
Assign the local equation numbers in all elements If the boolean argument is true then also store poi...
Definition: mesh.cc:765
virtual void read(std::ifstream &restart_file)
Read solution from restart file.
Definition: mesh.cc:1130
void set_consistent_pinned_values_for_continuation(ContinuationStorageScheme *const &continuation_stepper_pt)
Set consistent values for pinned data in continuation.
Definition: mesh.cc:2436
unsigned nhaloed_node()
Total number of haloed nodes in this Mesh.
Definition: mesh.h:1985
unsigned nexternal_haloed_node()
Total number of external haloed nodes in this Mesh.
Definition: mesh.h:2412
unsigned long assign_global_eqn_numbers(Vector< double * > &Dof_pt)
Assign the global equation numbers in the Data stored at the nodes and also internal element Data....
Definition: mesh.cc:677
virtual void distribute(OomphCommunicator *comm_pt, const Vector< unsigned > &element_domain, Vector< GeneralisedElement * > &deleted_element_pt, DocInfo &doc_info, const bool &report_stats, const bool &overrule_keep_as_halo_element_status)
Distribute the problem and doc; make this virtual to allow overloading for particular meshes where fu...
Definition: mesh.cc:4959
void null_external_halo_node(const unsigned &p, Node *nod_pt)
Null out specified external halo node (used when deleting duplicates)
Definition: mesh.cc:8569
Vector< GeneralisedElement * > root_haloed_element_pt(const unsigned &p)
Vector of pointers to root haloed elements in this Mesh whose non-halo counterpart is held on process...
Definition: mesh.h:1951
GeneralisedElement *& external_haloed_element_pt(const unsigned &p, const unsigned &e)
Access fct to the e-th external haloed element in this Mesh whose non-halo counterpart is held on pro...
Definition: mesh.h:2296
unsigned long nelement() const
Return number of elements in the mesh.
Definition: mesh.h:590
Vector< GeneralisedElement * > root_halo_element_pt(const unsigned &p)
Vector of pointers to root halo elements in this Mesh whose non-halo counterpart is held on processor...
Definition: mesh.h:1854
void merge_meshes(const Vector< Mesh * > &sub_mesh_pt)
Merge meshes. Note: This simply merges the meshes' elements and nodes (ignoring duplicates; no bounda...
Definition: mesh.cc:65
unsigned nexternal_halo_element()
Total number of external halo elements in this Mesh.
Definition: mesh.h:2222
virtual void dump(std::ofstream &dump_file, const bool &use_old_ordering=true) const
Dump the data in the mesh into a file for restart.
Definition: mesh.cc:1088
Vector< GeneralisedElement * > halo_element_pt(const unsigned &p)
Return vector of halo elements in this Mesh whose non-halo counterpart is held on processor p.
Definition: mesh.h:1740
A class to handle errors in the Newton solver.
Definition: problem.h:3057
bool linear_solver_error()
Access function to the error in the linear solver.
Definition: problem.h:3103
unsigned iterations()
Access function to Max. # of iterations performed when the Newton solver died.
Definition: problem.h:3110
double maxres()
Access function to Max. residual when Newton solver died.
Definition: problem.h:3116
Nodes are derived from Data, but, in addition, have a definite (Eulerian) position in a space of a gi...
Definition: nodes.h:906
double & x(const unsigned &i)
Return the i-th nodal coordinate.
Definition: nodes.h:1060
double & x_gen(const unsigned &k, const unsigned &i)
Reference to the generalised position x(k,i). ‘Type’: k; Coordinate direction: i.
Definition: nodes.h:1126
HangInfo *const & hanging_pt() const
Return pointer to hanging node data (this refers to the geometric hanging node status) (const version...
Definition: nodes.h:1228
void copy(Node *orig_node_pt)
Copy all nodal data from specified Node object.
Definition: nodes.cc:1916
void add_values_to_vector(Vector< double > &vector_of_values)
Add all data and time history values to the vector. Overloaded to add the position information as wel...
Definition: nodes.cc:2777
virtual void get_boundaries_pt(std::set< unsigned > *&boundaries_pt)
Return a pointer to set of mesh boundaries that this node occupies; this will be overloaded by Bounda...
Definition: nodes.h:1365
void position(Vector< double > &pos) const
Compute Vector of nodal positions either directly or via hanging node representation.
Definition: nodes.cc:2499
unsigned ndim() const
Return (Eulerian) spatial dimension of the node.
Definition: nodes.h:1054
void read_values_from_vector(const Vector< double > &vector_of_values, unsigned &index)
Read all data and time history values from the vector starting from index. On return the index will b...
Definition: nodes.cc:2832
void resize(const unsigned &n_value)
Resize the number of equations.
Definition: nodes.cc:2167
unsigned nposition_type() const
Number of coordinate types needed in the mapping between local and global coordinates.
Definition: nodes.h:1016
bool is_hanging() const
Test whether the node is geometrically hanging.
Definition: nodes.h:1285
double value(const unsigned &i) const
Return i-th value (dofs or pinned) at this node either directly or via hanging node representation....
Definition: nodes.cc:2408
An oomph-lib wrapper to the MPI_Comm communicator object. Just contains an MPI_Comm object (which is ...
Definition: communicator.h:54
std::ostream *& stream_pt()
Access function for the stream pointer.
An OomphLibError object which should be thrown when an run-time error is encountered....
An OomphLibWarning object which should be created as a temporary object to issue a warning....
A class that is used to assemble the residuals in parallel by overloading the get_all_vectors_and_mat...
A class that is used to define the functions used when assembling the derivatives of the residuals wi...
////////////////////////////////////////////////////////////////// //////////////////////////////////...
Definition: problem.h:154
virtual void actions_after_implicit_timestep()
Actions that should be performed after each implicit time step. This is needed when one wants to solv...
Definition: problem.h:1090
bool Always_take_one_newton_step
Boolean to indicate whether a Newton step should be taken even if the initial residuals are below the...
Definition: problem.h:2302
bool Jacobian_reuse_is_enabled
Is re-use of Jacobian in Newton iteration enabled? Default: false.
Definition: problem.h:621
double *& dof_pt(const unsigned &i)
Pointer to i-th dof in the problem.
Definition: problem.h:1855
virtual void actions_after_newton_solve()
Any actions that are to be performed after a complete Newton solve, e.g. post processing....
Definition: problem.h:1058
void parallel_sparse_assemble(const LinearAlgebraDistribution *const &dist_pt, Vector< int * > &column_or_row_index, Vector< int * > &row_or_column_start, Vector< double * > &value, Vector< unsigned > &nnz, Vector< double * > &residuals)
Helper method to assemble CRDoubleMatrices from distributed on multiple processors.
Definition: problem.cc:6534
void describe_dofs(std::ostream &out= *(oomph_info.stream_pt())) const
Function to describe the dofs in terms of the global equation number, i.e. what type of value (nodal ...
Definition: problem.cc:2446
virtual void sparse_assemble_row_or_column_compressed(Vector< int * > &column_or_row_index, Vector< int * > &row_or_column_start, Vector< double * > &value, Vector< unsigned > &nnz, Vector< double * > &residual, bool compressed_row_flag)
Protected helper function that is used to assemble the Jacobian matrix in the case when the storage i...
Definition: problem.cc:4462
bool Store_local_dof_pt_in_elements
Boolean to indicate whether local dof pointers should be stored in the elements.
Definition: problem.h:229
virtual void actions_before_newton_step()
Any actions that are to be performed before each individual Newton step. Most likely to be used for d...
Definition: problem.h:1073
void remove_duplicate_data(Mesh *const &mesh_pt, bool &actually_removed_some_data)
Private helper function to remove repeated data in external haloed elements in specified mesh....
Definition: problem.cc:2655
bool Bifurcation_detection
Boolean to control bifurcation detection via determinant of Jacobian.
Definition: problem.h:810
void adapt()
Adapt problem: Perform mesh adaptation for (all) refineable (sub)mesh(es), based on their own error e...
Definition: problem.h:2964
virtual void actions_before_newton_solve()
Any actions that are to be performed before a complete Newton solve (e.g. adjust boundary conditions)...
Definition: problem.h:1052
Vector< unsigned > First_el_for_assembly
First element to be assembled by given processor for non-distributed problem (only kept up to date wh...
Definition: problem.h:522
unsigned long assign_eqn_numbers(const bool &assign_local_eqn_numbers=true)
Assign all equation numbers for problem: Deals with global data (= data that isn't attached to any el...
Definition: problem.cc:2076
void refine_uniformly_aux(const Vector< unsigned > &nrefine_for_mesh, DocInfo &doc_info, const bool &prune)
Helper function to do compund refinement of (all) refineable (sub)mesh(es) uniformly as many times as...
Definition: problem.cc:15508
void build_global_mesh()
Build the global mesh by combining the all the submeshes. Note: The nodes boundary information refers...
Definition: problem.cc:1580
unsigned unrefine_uniformly()
Refine (all) refineable (sub)mesh(es) uniformly and rebuild problem. Return 0 for success,...
Definition: problem.cc:15894
void assign_initial_values_impulsive()
Initialise data and nodal positions to simulate impulsive start from initial configuration/solution.
Definition: problem.cc:11562
double & dof_current(const unsigned &i)
Access function to the current value of the i-th (local) dof at the start of a continuation step.
Definition: problem.h:1202
double Theta_squared
Value of the scaling parameter required so that the parameter occupies the desired proportion of the ...
Definition: problem.h:762
LinearAlgebraDistribution *const & dof_distribution_pt() const
Return the pointer to the dof distribution (read-only)
Definition: problem.h:1698
virtual void get_eigenproblem_matrices(CRDoubleMatrix &mass_matrix, CRDoubleMatrix &main_matrix, const double &shift=0.0)
Get the matrices required by a eigensolver. If the shift parameter is non-zero the second matrix will...
Definition: problem.cc:8429
Vector< double > Dof_derivative
Storage for the derivative of the problem variables wrt arc-length.
Definition: problem.h:794
virtual void sparse_assemble_row_or_column_compressed_with_lists(Vector< int * > &column_or_row_index, Vector< int * > &row_or_column_start, Vector< double * > &value, Vector< unsigned > &nnz, Vector< double * > &residual, bool compressed_row_flag)
Private helper function that is used to assemble the Jacobian matrix in the case when the storage is ...
Definition: problem.cc:4908
virtual void sparse_assemble_row_or_column_compressed_with_two_arrays(Vector< int * > &column_or_row_index, Vector< int * > &row_or_column_start, Vector< double * > &value, Vector< unsigned > &nnz, Vector< double * > &residual, bool compressed_row_flag)
Private helper function that is used to assemble the Jacobian matrix in the case when the storage is ...
Definition: problem.cc:6039
friend class BlockHopfLinearSolver
Definition: problem.h:165
void synchronise_dofs(const bool &do_halos, const bool &do_external_halos)
Synchronise the degrees of freedom by overwriting the haloed values with their non-halo counterparts ...
Definition: problem.cc:16525
virtual void actions_before_distribute()
Actions to be performed before a (mesh) distribution.
Definition: problem.h:1136
friend class FoldHandler
Definition: problem.h:156
Distributed_problem_matrix_distribution Dist_problem_matrix_distribution
The distributed matrix distribution method 1 - Automatic - the Problem distribution is employed,...
Definition: problem.h:976
DoubleVectorHaloScheme * Halo_scheme_pt
Pointer to the halo scheme for any global vectors that have the Dof_distribution.
Definition: problem.h:580
virtual void actions_after_change_in_global_parameter(double *const &parameter_pt)
Actions that are to be performed when the global parameter addressed by parameter_pt has been changed...
Definition: problem.h:1153
void p_refine_selected_elements(const Vector< unsigned > &elements_to_be_refined)
p-refine (one and only!) mesh by refining the elements identified by their numbers relative to the pr...
Definition: problem.cc:15222
void setup_dof_halo_scheme()
Function that is used to setup the halo scheme.
Definition: problem.cc:387
unsigned long ndof() const
Return the number of dofs.
Definition: problem.h:1704
void p_unrefine_uniformly(DocInfo &doc_info)
p-unrefine (all) p-refineable (sub)mesh(es) uniformly and rebuild problem.
Definition: problem.cc:16023
virtual void build_mesh()
Function to build the Problem's base mesh; this must be supplied by the user if they wish to use the ...
Definition: problem.h:1385
static bool Suppress_warning_about_actions_before_read_unstructured_meshes
Flag to allow suppression of warning messages re reading in unstructured meshes during restart.
Definition: problem.h:320
void solve_adjoint_eigenproblem(const unsigned &n_eval, Vector< std::complex< double >> &eigenvalue, Vector< DoubleVector > &eigenvector_real, Vector< DoubleVector > &eigenvector_imag, const bool &steady=true)
Solve an adjoint eigenvalue problem using the same procedure as solve_eigenproblem....
Definition: problem.cc:8363
bool Use_default_partition_in_load_balance
Flag to use "default partition" during load balance. Should only be set to true when run in validatio...
Definition: problem.h:517
void bifurcation_adapt_helper(unsigned &n_refined, unsigned &n_unrefined, const unsigned &bifurcation_type, const bool &actually_adapt=true)
A function that is used to adapt a bifurcation-tracking problem, which requires separate interpolatio...
Definition: problem.cc:13417
friend class AugmentedBlockFoldLinearSolver
Definition: problem.h:163
Vector< double > Elemental_assembly_time
Storage for assembly times (used for load balancing)
Definition: problem.h:576
bool Jacobian_has_been_computed
Has a Jacobian been computed (and can therefore be re-used if required)? Default: false.
Definition: problem.h:625
bool Bypass_increase_in_dof_check_during_pruning
Boolean to bypass check of increase in dofs during pruning.
Definition: problem.h:987
virtual void get_dvaluesdt(DoubleVector &f)
Get the time derivative of all values (using get_inverse_mass_matrix_times_residuals(....
Definition: problem.cc:3771
void initialise_dt(const double &dt)
Set all timesteps to the same value, dt, and assign weights for all timesteppers in the problem.
Definition: problem.cc:13294
void add_to_dofs(const double &lambda, const DoubleVector &increment_dofs)
Add lambda x incremenet_dofs[l] to the l-th dof.
Definition: problem.cc:3651
double Timestep_reduction_factor_after_nonconvergence
What it says: If temporally adaptive Newton solver fails to to converge, reduce timestep by this fact...
Definition: problem.h:2307
Vector< double > Dof_current
Storage for the present values of the variables.
Definition: problem.h:797
double Desired_proportion_of_arc_length
Proportion of the arc-length to taken by the parameter.
Definition: problem.h:754
AssemblyHandler *& assembly_handler_pt()
Return a pointer to the assembly handler object.
Definition: problem.h:1590
virtual void actions_after_implicit_timestep_and_error_estimation()
Actions that should be performed after each implicit time step. This is needed if your actions_after_...
Definition: problem.h:1095
void disable_mass_matrix_reuse()
Turn off recyling of the mass matrix in explicit timestepping schemes.
Definition: problem.cc:11895
EigenSolver * Default_eigen_solver_pt
Pointer to the default eigensolver.
Definition: problem.h:194
unsigned Nnewton_iter_taken
Actual number of Newton iterations taken during the most recent iteration.
Definition: problem.h:606
double * bifurcation_parameter_pt() const
Return pointer to the parameter that is used in the bifurcation detection. If we are not tracking a b...
Definition: problem.cc:10101
void copy_haloed_eqn_numbers_helper(const bool &do_halos, const bool &do_external_halos)
A private helper function to copy the haloed equation numbers into the halo equation numbers,...
Definition: problem.cc:17014
virtual void sparse_assemble_row_or_column_compressed_with_vectors_of_pairs(Vector< int * > &column_or_row_index, Vector< int * > &row_or_column_start, Vector< double * > &value, Vector< unsigned > &nnz, Vector< double * > &residual, bool compressed_row_flag)
Private helper function that is used to assemble the Jacobian matrix in the case when the storage is ...
Definition: problem.cc:5323
void send_data_to_be_sent_during_load_balancing(Vector< int > &send_n, Vector< double > &send_data, Vector< int > &send_displacement)
Load balance helper routine: Send data to other processors during load balancing.
Definition: problem.cc:19012
bool Time_adaptive_newton_crash_on_solve_fail
Bool to specify what to do if a Newton solve fails within a time adaptive solve. Default (false) is t...
Definition: problem.h:618
Problem()
Constructor: Allocate space for one time stepper and set all pointers to NULL and set defaults for al...
Definition: problem.cc:69
bool is_dparameter_calculated_analytically(double *const &parameter_pt)
Function to determine whether the parameter derivatives are calculated analytically.
Definition: problem.h:280
void flush_sub_meshes()
Flush the problem's collection of sub-meshes. Must be followed by call to rebuild_global_mesh().
Definition: problem.h:1359
bool & use_predictor_values_as_initial_guess()
Definition: problem.h:2099
unsigned Sparse_assembly_method
Flag to determine which sparse assembly method to use By default we use assembly by vectors of pairs.
Definition: problem.h:644
Time *& time_pt()
Return a pointer to the global time object.
Definition: problem.h:1524
void calculate_predictions()
Calculate predictions.
Definition: problem.cc:11719
Vector< unsigned > Last_el_plus_one_for_assembly
Last element (plus one) to be assembled by given processor for non-distributed problem (only kept up ...
Definition: problem.h:527
double * global_dof_pt(const unsigned &i)
Return a pointer to the dof, indexed by global equation number which may be haloed or stored locally....
Definition: problem.h:1794
virtual void actions_after_read_unstructured_meshes()
Actions that are to be performed before reading in restart data for problems involving unstructured b...
Definition: problem.h:1129
void copy(Problem *orig_problem_pt)
Copy Data values, nodal positions etc from specified problem. Note: This is not a copy constructor....
Definition: problem.cc:11928
virtual void get_jacobian(DoubleVector &residuals, DenseDoubleMatrix &jacobian)
Return the fully-assembled Jacobian and residuals for the problem Interface for the case when the Jac...
Definition: problem.cc:3922
void set_explicit_time_stepper_pt(ExplicitTimeStepper *const &explicit_time_stepper_pt)
Set the explicit timestepper for the problem. The function will automatically create or resize the Ti...
Definition: problem.cc:1673
virtual void actions_after_distribute()
Actions to be performed after a (mesh) distribution.
Definition: problem.h:1139
virtual void actions_before_implicit_timestep()
Actions that should be performed before each implicit time step. This is needed when one wants to sol...
Definition: problem.h:1084
LinearSolver * Linear_solver_pt
Pointer to the linear solver for the problem.
Definition: problem.h:176
bool Doc_time_in_distribute
Protected boolean flag to provide comprehensive timimings during problem distribution....
Definition: problem.h:640
unsigned Max_newton_iterations
Maximum number of Newton iterations.
Definition: problem.h:602
Vector< Vector< unsigned > > Sparse_assemble_with_arrays_previous_allocation
the number of elements in each row of a compressed matrix in the previous matrix assembly.
Definition: problem.h:670
virtual void actions_after_parameter_increase(double *const &parameter_pt)
Empty virtual function; provides hook to perform actions after the increase in the arclength paramete...
Definition: problem.h:1181
friend class HopfHandler
Definition: problem.h:158
friend class PitchForkHandler
Definition: problem.h:157
void calculate_continuation_derivatives(double *const &parameter_pt)
A function to calculate the derivatives wrt the arc-length. This version of the function actually doe...
Definition: problem.cc:9735
void store_current_dof_values()
Store the current values of the degrees of freedom.
Definition: problem.cc:8614
bool Problem_has_been_distributed
Has the problem been distributed amongst multiple processors?
Definition: problem.h:984
void synchronise_all_dofs()
Perform all required synchronisation in solvers.
Definition: problem.cc:16502
bool Empty_actions_before_read_unstructured_meshes_has_been_called
Boolean to indicate that empty actions_before_read_unstructured_meshes() function has been called.
Definition: problem.h:221
OomphCommunicator * communicator_pt()
access function to the oomph-lib communicator
Definition: problem.h:1266
bool Mass_matrix_has_been_computed
Has the mass matrix been computed (and can therefore be reused) Default: false.
Definition: problem.h:698
unsigned nglobal_data() const
Return the number of global data values.
Definition: problem.h:1716
virtual void actions_before_adapt()
Actions that are to be performed before a mesh adaptation. These might include removing any additiona...
Definition: problem.h:1042
void newton_solve()
Use Newton method to solve the problem.
Definition: problem.cc:8803
bool First_jacobian_sign_change
Boolean to indicate whether a sign change has occured in the Jacobian.
Definition: problem.h:816
void refine_distributed_base_mesh(Vector< Vector< Vector< unsigned >>> &to_be_refined_on_each_root, const unsigned &max_level_overall)
Load balance helper routine: refine each new base (sub)mesh based upon the elements to be refined wit...
Definition: problem.cc:20053
void calculate_continuation_derivatives_fd_helper(double *const &parameter_pt)
A function that performs the guts of the continuation derivative calculation in arc-length continuati...
Definition: problem.cc:10024
double Continuation_direction
The direction of the change in parameter that will ensure that a branch is followed in one direction ...
Definition: problem.h:769
unsigned Parallel_sparse_assemble_previous_allocation
The amount of data allocated during the previous parallel sparse assemble. Used to optimise the next ...
Definition: problem.h:980
void enable_mass_matrix_reuse()
Enable recycling of the mass matrix in explicit timestepping schemes. Useful for timestepping on fixe...
Definition: problem.cc:11870
void steady_newton_solve(unsigned const &max_adapt=0)
Solve a steady problem using adaptive Newton's method, but in the context of an overall unstady probl...
Definition: problem.cc:9312
bool Scale_arc_length
Boolean to control whether arc-length should be scaled.
Definition: problem.h:751
double doubly_adaptive_unsteady_newton_solve_helper(const double &dt, const double &epsilon, const unsigned &max_adapt, const unsigned &suppress_resolve_after_spatial_adapt, const bool &first, const bool &shift=true)
Private helper function that actually performs the unsteady "doubly" adaptive Newton solve....
Definition: problem.cc:11442
bool Empty_actions_after_read_unstructured_meshes_has_been_called
Boolean to indicate that empty actions_after_read_unstructured_meshes() function has been called.
Definition: problem.h:225
virtual void get_residuals(DoubleVector &residuals)
Return the fully-assembled residuals Vector for the problem: Virtual so it can be overloaded in for m...
Definition: problem.cc:3801
Vector< GeneralisedElement * > Base_mesh_element_pt
Vector to store the correspondence between a root element and its element number within the global me...
Definition: problem.h:551
double & dof(const unsigned &i)
i-th dof in the problem
Definition: problem.h:1843
void get_fd_jacobian(DoubleVector &residuals, DenseMatrix< double > &jacobian)
Return the fully-assembled Jacobian and residuals, generated by finite differences.
Definition: problem.cc:7801
virtual void sparse_assemble_row_or_column_compressed_with_two_vectors(Vector< int * > &column_or_row_index, Vector< int * > &row_or_column_start, Vector< double * > &value, Vector< unsigned > &nnz, Vector< double * > &residual, bool compressed_row_flag)
Private helper function that is used to assemble the Jacobian matrix in the case when the storage is ...
Definition: problem.cc:5676
virtual Problem * make_copy()
Make and return a pointer to the copy of the problem. A virtual function that must be filled in by th...
Definition: problem.cc:12074
double Maximum_dt
Maximum desired dt.
Definition: problem.h:712
unsigned long set_timestepper_for_all_data(TimeStepper *const &time_stepper_pt, const bool &preserve_existing_data=false)
Set all problem data to have the same timestepper (timestepper_pt) Return the new number of dofs in t...
Definition: problem.cc:11635
void activate_pitchfork_tracking(double *const &parameter_pt, const DoubleVector &symmetry_vector, const bool &block_solve=true)
Turn on pitchfork tracking using the augmented system specified in the PitchForkHandler class....
Definition: problem.cc:10208
virtual void dump(std::ofstream &dump_file) const
Dump refinement pattern of all refineable meshes and all generic Problem data to file for restart.
Definition: problem.cc:12092
bool Use_finite_differences_for_continuation_derivatives
Boolean to specify which scheme to use to calculate the continuation derivatievs.
Definition: problem.h:823
bool Arc_length_step_taken
Boolean to indicate whether an arc-length step has been taken.
Definition: problem.h:819
void set_pinned_values_to_zero()
Set all pinned values to zero. Used to set boundary conditions to be homogeneous in the copy of the p...
Definition: problem.cc:4282
bool Default_set_initial_condition_called
Has default set_initial_condition function been called? Default: false.
Definition: problem.h:214
void get_bifurcation_eigenfunction(Vector< DoubleVector > &eigenfunction)
Return the eigenfunction calculated as part of a bifurcation tracking process. If we are not tracking...
Definition: problem.cc:10111
double Relaxation_factor
Relaxation fator for Newton method (only a fractional Newton correction is applied if this is less th...
Definition: problem.h:595
void add_time_stepper_pt(TimeStepper *const &time_stepper_pt)
Add a timestepper to the problem. The function will automatically create or resize the Time object so...
Definition: problem.cc:1632
void refine_selected_elements(const Vector< unsigned > &elements_to_be_refined)
Refine (one and only!) mesh by splitting the elements identified by their numbers relative to the pro...
Definition: problem.cc:14961
void prune_halo_elements_and_nodes(DocInfo &doc_info, const bool &report_stats)
(Irreversibly) prune halo(ed) elements and nodes, usually after another round of refinement,...
Definition: problem.cc:1140
virtual void get_inverse_mass_matrix_times_residuals(DoubleVector &Mres)
Return the residual vector multiplied by the inverse mass matrix Virtual so that it can be overloaded...
Definition: problem.cc:3666
void check_halo_schemes()
Check the halo/haloed node/element schemes.
Definition: problem.h:2221
double Parameter_current
Storage for the present value of the global parameter.
Definition: problem.h:775
void assign_eigenvector_to_dofs(DoubleVector &eigenvector)
Assign the eigenvector passed to the function to the dofs in the problem so that it can be output by ...
Definition: problem.cc:8723
@ Uniform_matrix_distribution
Definition: problem.h:851
@ Default_matrix_distribution
Definition: problem.h:849
@ Problem_matrix_distribution
Definition: problem.h:850
unsigned setup_element_count_per_dof()
Function that populates the Element_counter_per_dof vector with the number of elements that contribut...
Definition: problem.cc:231
LinearSolver *& mass_matrix_solver_for_explicit_timestepper_pt()
Return a pointer to the linear solver object used for explicit time stepping.
Definition: problem.h:1499
OomphCommunicator * Communicator_pt
The communicator for this problem.
Definition: problem.h:1262
double Newton_solver_tolerance
The Tolerance below which the Newton Method is deemed to have converged.
Definition: problem.h:599
void get_flat_packed_refinement_pattern_for_load_balancing(const Vector< unsigned > &old_domain_for_base_element, const Vector< unsigned > &new_domain_for_base_element, const unsigned &max_refinement_level_overall, std::map< unsigned, Vector< unsigned >> &flat_packed_refinement_info_for_root)
Get flat-packed refinement pattern for each root element in current mesh (labeled by unique number of...
Definition: problem.cc:19898
void set_consistent_pinned_values_for_continuation()
Private helper function that is used to set the appropriate pinned values for continuation.
Definition: problem.cc:10486
void activate_bifurcation_tracking(double *const &parameter_pt, const DoubleVector &eigenvector, const bool &block_solve=true)
Activate generic bifurcation tracking for a single (real) eigenvalue where the initial guess for the ...
Definition: problem.cc:10149
bool Discontinuous_element_formulation
Is the problem a discontinuous one, i.e. can the elemental contributions be treated independently....
Definition: problem.h:703
bool Doc_imbalance_in_parallel_assembly
Boolean to switch on assessment of load imbalance in parallel assembly of distributed problem.
Definition: problem.h:513
long synchronise_eqn_numbers(const bool &assign_local_eqn_numbers=true)
Classify any non-classified nodes into halo/haloed and synchronise equation numbers....
Definition: problem.cc:16768
void create_new_linear_algebra_distribution(LinearAlgebraDistribution *&dist_pt)
Get new linear algebra distribution (you're in charge of deleting it!)
Definition: problem.cc:300
void p_refine_uniformly_aux(const Vector< unsigned > &nrefine_for_mesh, DocInfo &doc_info, const bool &prune)
Helper function to do compund p-refinement of (all) p-refineable (sub)mesh(es) uniformly as many time...
Definition: problem.cc:15655
void calculate_continuation_derivatives_helper(const DoubleVector &z)
A function that performs the guts of the continuation derivative calculation in arc length continuati...
Definition: problem.cc:9932
Vector< Problem * > Copy_of_problem_pt
Vector of pointers to copies of the problem used in adaptive bifurcation tracking problems (ALH: TEMP...
Definition: problem.h:237
Vector< unsigned > distribute(const Vector< unsigned > &element_partition, DocInfo &doc_info, const bool &report_stats=false)
Distribute the problem and doc, using the specified partition; returns a vector which details the par...
Definition: problem.cc:491
Mesh * Mesh_pt
The mesh pointer.
Definition: problem.h:170
double Parameter_derivative
Storage for the derivative of the global parameter wrt arc-length.
Definition: problem.h:772
void add_eigenvector_to_dofs(const double &epsilon, const DoubleVector &eigenvector)
Add the eigenvector passed to the function scaled by the constat epsilon to the dofs in the problem s...
Definition: problem.cc:8760
Vector< Data * > Global_data_pt
Vector of global data: "Nobody" (i.e. none of the elements etc.) is "in charge" of this Data so it wo...
Definition: problem.h:428
void recompute_load_balanced_assembly()
Helper function to re-assign the first and last elements to be assembled by each processor during par...
Definition: problem.cc:1776
Vector< double * > Dof_pt
Vector of pointers to dofs.
Definition: problem.h:557
void p_adapt()
p-adapt problem: Perform mesh adaptation for (all) refineable (sub)mesh(es), based on their own error...
Definition: problem.h:2986
double DTSF_max_increase
Maximum possible increase of dt between time-steps in adaptive schemes.
Definition: problem.h:716
bool Must_recompute_load_balance_for_assembly
Boolean indicating that the division of elements over processors during the assembly process must be ...
Definition: problem.h:532
virtual void shift_time_values()
Shift all values along to prepare for next timestep.
Definition: problem.cc:11697
double Target_error_safety_factor
Safety factor to ensure we are aiming for a target error, TARGET, that is below our tolerance: TARGET...
Definition: problem.h:738
Mesh *& mesh_pt()
Return a pointer to the global mesh.
Definition: problem.h:1300
void set_default_first_and_last_element_for_assembly()
Set default first and last elements for parallel assembly of non-distributed problem.
Definition: problem.cc:1700
bool Keep_temporal_error_below_tolerance
Boolean to decide if a timestep is to be rejected if the error estimate post-solve (computed by globa...
Definition: problem.h:2314
bool Shut_up_in_newton_solve
Boolean to indicate if all output is suppressed in Problem::newton_solve(). Defaults to false.
Definition: problem.h:2296
void set_dofs(const DoubleVector &dofs)
Set the values of the dofs.
Definition: problem.cc:3498
void setup_base_mesh_info_after_pruning()
Helper function to re-setup the Base_mesh enumeration (used during load balancing) after pruning.
Definition: problem.cc:20165
Vector< Mesh * > Sub_mesh_pt
Vector of pointers to submeshes.
Definition: problem.h:173
Vector< double > Max_res
Maximum residuals at start and after each newton iteration.
Definition: problem.h:609
EigenSolver * Eigen_solver_pt
Pointer to the eigen solver for the problem.
Definition: problem.h:185
bool Bisect_to_find_bifurcation
Boolean to control wheter bisection is used to located bifurcation.
Definition: problem.h:813
void explicit_timestep(const double &dt, const bool &shift_values=true)
Take an explicit timestep of size dt and optionally shift any stored values of the time history.
Definition: problem.cc:10948
void delete_all_external_storage()
Wrapper function to delete external storage for each submesh of the problem.
Definition: problem.cc:16418
friend class BlockPitchForkLinearSolver
Definition: problem.h:162
double DTSF_min_decrease
Minimum allowed decrease of dt between time-steps in adaptive schemes. Lower scaling values will reje...
Definition: problem.h:721
virtual void symmetrise_eigenfunction_for_adaptive_pitchfork_tracking()
Virtual function that is used to symmetrise the problem so that the current solution exactly satisfie...
Definition: problem.cc:10079
double Minimum_dt_but_still_proceed
If Minimum_dt_but_still_proceed positive, then dt will not be reduced below this value during adaptiv...
Definition: problem.h:745
double adaptive_unsteady_newton_solve(const double &dt_desired, const double &epsilon)
Attempt to advance timestep by dt_desired. If the solution fails the timestep will be halved until co...
Definition: problem.cc:11086
unsigned Desired_newton_iterations_ds
The desired number of Newton Steps to reach convergence at each step along the arc.
Definition: problem.h:804
void rebuild_global_mesh()
If one of the submeshes has changed (e.g. by mesh adaptation) we need to update the global mesh....
Definition: problem.cc:1620
ExplicitTimeStepper *& explicit_time_stepper_pt()
Return a pointer to the explicit timestepper.
Definition: problem.h:1575
void activate_hopf_tracking(double *const &parameter_pt, const bool &block_solve=true)
Turn on Hopf bifurcation tracking using the augmented system specified in the HopfHandler class....
Definition: problem.cc:10238
Vector< double * > Halo_dof_pt
Storage for the halo degrees of freedom (only required) when accessing via the global equation number...
Definition: problem.h:584
void deactivate_bifurcation_tracking()
Deactivate all bifuraction tracking, by reseting the assembly handler to the default.
Definition: problem.h:2458
void globally_convergent_line_search(const Vector< double > &x_old, const double &half_residual_squared_old, DoubleVector &gradient, DoubleVector &newton_dir, double &half_residual_squared, const double &stpmax)
Line search helper for globally convergent Newton method.
Definition: problem.cc:9166
void get_hessian_vector_products(DoubleVectorWithHaloEntries const &Y, Vector< DoubleVectorWithHaloEntries > const &C, Vector< DoubleVectorWithHaloEntries > &product)
Return the product of the global hessian (derivative of Jacobian matrix with respect to all variables...
Definition: problem.cc:7963
virtual double global_temporal_error_norm()
Function to calculate a global error norm, used in adaptive timestepping to control the change in tim...
Definition: problem.h:1250
@ Perform_assembly_using_two_arrays
Definition: problem.h:653
@ Perform_assembly_using_maps
Definition: problem.h:651
@ Perform_assembly_using_two_vectors
Definition: problem.h:650
@ Perform_assembly_using_vectors_of_pairs
Definition: problem.h:649
@ Perform_assembly_using_lists
Definition: problem.h:652
int Sign_of_jacobian
Storage for the sign of the global Jacobian.
Definition: problem.h:765
std::map< GeneralisedElement *, unsigned > Base_mesh_element_number_plus_one
Map which stores the correspondence between a root element and its element number (plus one) within t...
Definition: problem.h:541
double Max_residuals
Maximum desired residual: if the maximum residual exceeds this value, the program will exit.
Definition: problem.h:613
unsigned nsub_mesh() const
Return number of submeshes.
Definition: problem.h:1343
double & time()
Return the current value of continuous time.
Definition: problem.cc:11594
unsigned self_test()
Self-test: Check meshes and global data. Return 0 for OK.
Definition: problem.cc:13339
unsigned ntime_stepper() const
Return the number of time steppers.
Definition: problem.h:1710
void activate_fold_tracking(double *const &parameter_pt, const bool &block_solve=true)
Turn on fold tracking using the augmented system specified in the FoldHandler class....
Definition: problem.cc:10123
bool Use_globally_convergent_newton_method
Use the globally convergent newton method.
Definition: problem.h:217
LinearSolver * Default_linear_solver_pt
Pointer to the default linear solver.
Definition: problem.h:191
double Minimum_ds
Minimum desired value of arc-length.
Definition: problem.h:807
void get_data_to_be_sent_during_load_balancing(const Vector< unsigned > &element_domain_on_this_proc, Vector< int > &send_n, Vector< double > &send_data, Vector< int > &send_displacement, Vector< unsigned > &old_domain_for_base_element, Vector< unsigned > &new_domain_for_base_element, unsigned &max_refinement_level_overall)
Load balance helper routine: Get data to be sent to other processors during load balancing and other ...
Definition: problem.cc:19342
void restore_dof_values()
Restore the stored values of the degrees of freedom.
Definition: problem.cc:8660
double Numerical_zero_for_sparse_assembly
A tolerance used to determine whether the entry in a sparse matrix is zero. If it is then storage nee...
Definition: problem.h:674
double FD_step_used_in_get_hessian_vector_products
Definition: problem.h:688
virtual void partition_global_mesh(Mesh *&global_mesh_pt, DocInfo &doc_info, Vector< unsigned > &element_domain, const bool &report_stats=false)
Partition the global mesh, return vector specifying the processor number for each element....
Definition: problem.cc:965
unsigned Sparse_assemble_with_arrays_initial_allocation
the number of elements to initially allocate for a matrix row within the sparse_assembly_with_two_arr...
Definition: problem.h:661
void bifurcation_adapt_doc_errors(const unsigned &bifurcation_type)
A function that is used to document the errors used in the adaptive solution of bifurcation problems.
Definition: problem.cc:13711
double Ds_current
Storage for the current step value.
Definition: problem.h:800
virtual void set_initial_condition()
Set initial condition (incl previous timesteps). We need to establish this interface because I....
Definition: problem.h:1218
LinearSolver * Mass_matrix_solver_for_explicit_timestepper_pt
Pointer to the linear solver used for explicit time steps (this is likely to be different to the line...
Definition: problem.h:182
void get_all_halo_data(std::map< unsigned, double * > &map_of_halo_data)
Get pointers to all possible halo data indexed by global equation number in a map.
Definition: problem.cc:16440
void load_balance()
Balance the load of a (possibly non-uniformly refined) problem that has already been distributed,...
Definition: problem.h:1402
double & dof_derivative(const unsigned &i)
Access function to the derivative of the i-th (local) dof with respect to the arc length,...
Definition: problem.h:1188
double Minimum_dt
Minimum desired dt: if dt falls below this value, exit.
Definition: problem.h:709
TimeStepper *& time_stepper_pt()
Access function for the pointer to the first (presumably only) timestepper.
Definition: problem.h:1544
double arc_length_step_solve(double *const &parameter_pt, const double &ds, const unsigned &max_adapt=0)
Solve a steady problem using arc-length continuation, when the parameter that becomes a variable corr...
Definition: problem.cc:10314
virtual void actions_after_adapt()
Actions that are to be performed after a mesh adaptation.
Definition: problem.h:1045
void p_refine_uniformly()
p-refine (all) p-refineable (sub)mesh(es) uniformly and rebuild problem
Definition: problem.h:2842
bool Use_continuation_timestepper
Boolean to control original or new storage of dof stuff.
Definition: problem.h:778
void calculate_continuation_derivatives_fd(double *const &parameter_pt)
A function to calculate the derivatives with respect to the arc-length required for continuation by f...
Definition: problem.cc:9838
LinearAlgebraDistribution * Dof_distribution_pt
The distribution of the DOFs in this problem. This object is created in the Problem constructor and s...
Definition: problem.h:463
void refine_uniformly()
Refine (all) refineable (sub)mesh(es) uniformly and rebuild problem.
Definition: problem.h:2745
static ContinuationStorageScheme Continuation_time_stepper
Storage for the single static continuation timestorage object.
Definition: problem.h:781
bool Problem_is_nonlinear
Boolean flag indicating if we're dealing with a linear or nonlinear Problem – if set to false the New...
Definition: problem.h:631
virtual void sparse_assemble_row_or_column_compressed_with_maps(Vector< int * > &column_or_row_index, Vector< int * > &row_or_column_start, Vector< double * > &value, Vector< unsigned > &nnz, Vector< double * > &residual, bool compressed_row_flag)
Private helper function that is used to assemble the Jacobian matrix in the case when the storage is ...
Definition: problem.cc:4562
void send_refinement_info_helper(Vector< unsigned > &old_domain_for_base_element, Vector< unsigned > &new_domain_for_base_element, const unsigned &max_refinement_level_overall, std::map< unsigned, Vector< unsigned >> &refinement_info_for_root_local, Vector< Vector< Vector< unsigned >>> &refinement_info_for_root_elements)
Send refinement information between processors.
Definition: problem.cc:18391
AssemblyHandler * Default_assembly_handler_pt
Pointer to the default assembly handler.
Definition: problem.h:197
void get_my_eqns(AssemblyHandler *const &assembly_handler_pt, const unsigned &el_lo, const unsigned &el_hi, Vector< unsigned > &my_eqns)
Helper method that returns the (unique) global equations to which the elements in the range el_lo to ...
Definition: problem.cc:6487
virtual void read(std::ifstream &restart_file, bool &unsteady_restart)
Read refinement pattern of all refineable meshes and refine them accordingly, then read all Data and ...
Definition: problem.cc:12314
virtual ~Problem()
Virtual destructor to clean up memory.
Definition: problem.cc:182
void get_dofs(DoubleVector &dofs) const
Return the vector of dofs, i.e. a vector containing the current values of all unknowns.
Definition: problem.cc:2566
Time * Time_pt
Pointer to global time for the problem.
Definition: problem.h:200
DoubleVectorWithHaloEntries Element_count_per_dof
Counter that records how many elements contribute to each dof. Used to determine the (discrete) arc-l...
Definition: problem.h:563
Data *& global_data_pt(const unsigned &i)
Return a pointer to the the i-th global data object.
Definition: problem.h:1673
virtual void actions_before_newton_convergence_check()
Any actions that are to be performed before the residual is checked in the Newton method,...
Definition: problem.h:1068
unsigned newton_solve_continuation(double *const &parameter_pt)
Perform a basic arc-length continuation step using Newton's method. Returns number of Newton steps ta...
Definition: problem.cc:9396
double Max_permitted_error_for_halo_check
Threshold for error throwing in Problem::check_halo_schemes()
Definition: problem.h:2280
unsigned Sparse_assemble_with_arrays_allocation_increment
the number of elements to add to a matrix row when the initial allocation is exceeded within the spar...
Definition: problem.h:666
void reset_assembly_handler_to_default()
Reset the system to the standard non-augemented state.
Definition: problem.cc:10295
virtual void actions_after_newton_step()
Any actions that are to be performed after each individual Newton step. Most likely to be used for di...
Definition: problem.h:1078
bool Pause_at_end_of_sparse_assembly
Protected boolean flag to halt program execution during sparse assemble process to assess peak memory...
Definition: problem.h:636
void remove_null_pointers_from_external_halo_node_storage()
Consolidate external halo node storage by removing nulled out pointers in external halo and haloed sc...
Definition: problem.cc:3265
Vector< double > * Saved_dof_pt
Pointer to vector for backup of dofs.
Definition: problem.h:210
void unsteady_newton_solve(const double &dt)
Advance time by dt and solve by Newton's method. This version always shifts time values.
Definition: problem.cc:10983
AssemblyHandler * Assembly_handler_pt
Definition: problem.h:188
void get_all_error_estimates(Vector< Vector< double >> &elemental_error)
Return the error estimates computed by (all) refineable (sub)mesh(es) in the elemental_error structur...
Definition: problem.cc:14690
virtual void actions_before_read_unstructured_meshes()
Actions that are to be performed before reading in restart data for problems involving unstructured b...
Definition: problem.h:1115
Vector< TimeStepper * > Time_stepper_pt
The Vector of time steppers (there could be many different ones in multiphysics problems)
Definition: problem.h:204
void doc_errors()
Get max and min error for all elements in submeshes.
Definition: problem.h:3031
bool Mass_matrix_reuse_is_enabled
Is re-use of the mass matrix in explicit timestepping enabled Default:false.
Definition: problem.h:694
void get_derivative_wrt_global_parameter(double *const &parameter_pt, DoubleVector &result)
Get the derivative of the entire residuals vector wrt a global parameter, used in continuation proble...
Definition: problem.cc:7865
void adapt_based_on_error_estimates(unsigned &n_refined, unsigned &n_unrefined, Vector< Vector< double >> &elemental_error)
Adapt problem: Perform mesh adaptation for (all) refineable (sub)mesh(es), based on the error estimat...
Definition: problem.cc:14591
bool distributed() const
If we have MPI return the "problem has been distributed" flag, otherwise it can't be distributed so r...
Definition: problem.h:828
bool are_hessian_products_calculated_analytically()
Function to determine whether the hessian products are calculated analytically.
Definition: problem.h:306
bool does_pointer_correspond_to_problem_data(double *const &parameter_pt)
Return a boolean flag to indicate whether the pointer parameter_pt refers to values stored in a Data ...
Definition: problem.cc:9866
ExplicitTimeStepper * Explicit_time_stepper_pt
Pointer to a single explicit timestepper.
Definition: problem.h:207
double arc_length_step_solve_helper(double *const &parameter_pt, const double &ds, const unsigned &max_adapt)
Private helper function that actually contains the guts of the arc-length stepping,...
Definition: problem.cc:10535
void solve_eigenproblem(const unsigned &n_eval, Vector< std::complex< double >> &alpha, Vector< double > &beta, Vector< DoubleVector > &eigenvector_real, Vector< DoubleVector > &eigenvector_imag, const bool &steady=true)
Solve an eigenproblem as assembled by the Problem's constituent elements. Calculate (at least) n_eval...
Definition: problem.cc:8232
bool Use_predictor_values_as_initial_guess
Use values from the time stepper predictor as an initial guess.
Definition: problem.h:232
RefineableElements are FiniteElements that may be subdivided into children to provide a better local ...
Tree * tree_pt()
Access function: Pointer to quadtree representation of this element.
virtual RefineableElement * root_element_pt()
Pointer to the root element in refinement hierarchy (must be implemented in specific elements that do...
Base class for refineable meshes. Provides standardised interfaces for the following standard mesh ad...
General SolidMesh class.
Definition: mesh.h:2562
A Class for nodes that deform elastically (i.e. position is an unknown in the problem)....
Definition: nodes.h:1686
void copy(SolidNode *orig_node_pt)
Copy nodal positions and associated data from specified node object.
Definition: nodes.cc:3433
Data *const & variable_position_pt() const
Pointer to variable_position data (const version)
Definition: nodes.h:1765
bool position_is_pinned(const unsigned &i)
Test whether the i-th coordinate is pinned, 0: false; 1: true.
Definition: nodes.h:1803
/////////////////////////////////////////////////////////////////////// /////////////////////////////...
Definition: spines.h:613
//////////////////////////////////////////////////////////////////////////// ////////////////////////...
////////////////////////////////////////////////////////////////////// //////////////////////////////...
Definition: timesteppers.h:231
virtual unsigned ndt() const =0
Number of timestep increments that are required by the scheme.
virtual void set_weights()=0
Function to set the weights for present timestep (don't need to pass present timestep or previous tim...
virtual void set_predictor_weights()
Set the weights for the predictor previous timestep (currently empty – overwrite for specific scheme)
Definition: timesteppers.h:630
virtual void actions_before_timestep(Problem *problem_pt)
Interface for any actions that need to be performed before a time step.
Definition: timesteppers.h:662
virtual void actions_after_timestep(Problem *problem_pt)
Interface for any actions that need to be performed after a time step.
Definition: timesteppers.h:666
void make_steady()
Function to make the time stepper temporarily steady. This is trivially achieved by setting all the w...
Definition: timesteppers.h:374
virtual void undo_make_steady()
Reset the is_steady status of a specific TimeStepper to its default and re-assign the weights.
Definition: timesteppers.h:482
ExplicitTimeStepper * explicit_predictor_pt()
Get the pointer to the explicit timestepper to use as a predictor in adaptivity if Predict_by_explici...
Definition: timesteppers.h:403
void update_predicted_time(const double &new_time)
Set the time that the current predictions apply for, only needed for paranoid checks when doing Predi...
Definition: timesteppers.h:417
bool is_steady() const
Flag to indicate if a timestepper has been made steady (possibly temporarily to switch off time-depen...
Definition: timesteppers.h:389
virtual void set_error_weights()
Set the weights for the error computation, (currently empty – overwrite for specific scheme)
Definition: timesteppers.h:642
Time *const & time_pt() const
Access function for the pointer to time (const version)
Definition: timesteppers.h:572
Class to keep track of discrete/continous time. It is essential to have a single Time object when usi...
Definition: timesteppers.h:63
double & dt(const unsigned &t=0)
Return the value of the t-th stored timestep (t=0: present; t>0: previous).
Definition: timesteppers.h:136
double & time()
Return the current value of the continuous time.
Definition: timesteppers.h:123
void shift_dt()
Update all stored values of dt by shifting each value along the array. This function must be called b...
Definition: timesteppers.h:174
void initialise_dt(const double &dt_)
Set all timesteps to the same value, dt.
Definition: timesteppers.h:99
unsigned ndt() const
Return the number of timesteps stored.
Definition: timesteppers.h:129
void resize(const unsigned &n_dt)
Resize the vector holding the number of previous timesteps and initialise the new values to zero.
Definition: timesteppers.h:93
///////////////////////////////////////////////////////////////// ///////////////////////////////////...
void refine_base_mesh(Vector< Vector< unsigned >> &to_be_refined)
Refine base mesh according to specified refinement pattern.
unsigned uniform_refinement_level_when_pruned() const
Level to which the mesh was uniformly refined when it was pruned (const version)
virtual void get_refinement_levels(unsigned &min_refinement_level, unsigned &max_refinement_level)
Get max/min refinement levels in mesh.
void refine_uniformly(DocInfo &doc_info)
Refine mesh uniformly and doc process.
TreeRoot is a Tree that forms the root of a (recursive) tree. The "root node" is special as it holds ...
Definition: tree.h:324
void stick_all_tree_nodes_into_vector(Vector< Tree * > &)
Traverse and stick pointers to all "nodes" into Vector.
Definition: tree.cc:277
RefineableElement * object_pt() const
Return the pointer to the object (RefineableElement) represented by the tree.
Definition: tree.h:88
void stick_leaves_into_vector(Vector< Tree * > &)
Traverse tree and stick pointers to leaf "nodes" (only) into Vector.
Definition: tree.cc:255
TreeRoot *& root_pt()
Return pointer to root of the tree.
Definition: tree.h:141
Base class for triangle meshes (meshes made of 2D triangle elements). Note: we choose to template Tri...
Definition: triangle_mesh.h:52
void dump_triangulateio(std::ostream &dump_file)
Dump the triangulateio structure to a dump file and record boundary coordinates of boundary nodes.
virtual void read_distributed_info_for_restart(std::istream &restart_file)
Virtual function that is used to read info. related with distributed triangle meshes.
bool use_triangulateio_restart() const
const access for Use_triangulateio_restart.
void remesh_from_triangulateio(std::istream &restart_file)
Regenerate the mesh from a dumped triangulateio file and dumped boundary coordinates of boundary node...
virtual void dump_distributed_info_for_restart(std::ostream &dump_file)
Virtual function that is used to dump info. related with distributed triangle meshes.
virtual void reestablish_distribution_info_for_restart(OomphCommunicator *comm_pt, std::istream &restart_file)
Virtual function used to re-establish any additional info. related with the distribution after a re-s...
void initialise(const _Tp &__value)
Iterate over all values and set to the desired value.
Definition: Vector.h:167
std::string string(const unsigned &i)
Return the i-th string or "" if the relevant string hasn't been defined.
bool Doc_comprehensive_timings
Global boolean to switch on comprehensive timing – can probably be declared const false when developm...
unsigned Max_newton_iterations
Maximum number of newton iterations.
Definition: elements.cc:1682
void partition_distributed_mesh(Problem *problem_pt, const unsigned &objective, Vector< unsigned > &element_domain_on_this_proc, const bool &bypass_metis=false)
Use METIS to assign each element in an already-distributed mesh to a domain. On return,...
void partition_mesh(Problem *problem_pt, const unsigned &ndomain, const unsigned &objective, Vector< unsigned > &element_domain)
Use METIS to assign each element to a domain. On return, element_domain[ielem] contains the number of...
std::string to_string(T object, unsigned float_precision=8)
Conversion function that should work for anything with operator<< defined (at least all basic types).
void clean_up_memory()
Clean up function that deletes anything dynamically allocated in this namespace.
void setup()
Setup terminate helper.
double timer()
returns the time in seconds after some point in past
std::string convert_secs_to_formatted_string(const double &time_in_sec)
Returns a nicely formatted string from an input time in seconds; the format depends on the size of ti...
//////////////////////////////////////////////////////////////////// ////////////////////////////////...
void pause(std::string message)
Pause and display message.
OomphInfo oomph_info
Single (global) instantiation of the OomphInfo object – this is used throughout the library as a "rep...