problem.cc
Go to the documentation of this file.
1 // LIC// ====================================================================
2 // LIC// This file forms part of oomph-lib, the object-oriented,
3 // LIC// multi-physics finite-element library, available
4 // LIC// at http://www.oomph-lib.org.
5 // LIC//
6 // LIC// Copyright (C) 2006-2023 Matthias Heil and Andrew Hazel
7 // LIC//
8 // LIC// This library is free software; you can redistribute it and/or
9 // LIC// modify it under the terms of the GNU Lesser General Public
10 // LIC// License as published by the Free Software Foundation; either
11 // LIC// version 2.1 of the License, or (at your option) any later version.
12 // LIC//
13 // LIC// This library is distributed in the hope that it will be useful,
14 // LIC// but WITHOUT ANY WARRANTY; without even the implied warranty of
15 // LIC// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 // LIC// Lesser General Public License for more details.
17 // LIC//
18 // LIC// You should have received a copy of the GNU Lesser General Public
19 // LIC// License along with this library; if not, write to the Free Software
20 // LIC// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
21 // LIC// 02110-1301 USA.
22 // LIC//
23 // LIC// The authors may be contacted at oomph-lib@maths.man.ac.uk.
24 // LIC//
25 // LIC//====================================================================
26 
27 #ifdef OOMPH_HAS_MPI
28 #include "mpi.h"
29 #endif
30 
31 #include <list>
32 #include <algorithm>
33 #include <string>
34 
35 #include "oomph_utilities.h"
36 #include "problem.h"
37 #include "timesteppers.h"
38 #include "explicit_timesteppers.h"
40 #include "refineable_mesh.h"
41 #include "triangle_mesh.h"
42 #include "linear_solver.h"
43 #include "eigen_solver.h"
44 #include "assembly_handler.h"
45 #include "dg_elements.h"
46 #include "partitioning.h"
47 #include "spines.h"
48 
49 // Include to fill in additional_setup_shared_node_scheme() function
51 
52 
53 namespace oomph
54 {
55  /// ///////////////////////////////////////////////////////////////
56  // Non-inline functions for the problem class
57  /// ///////////////////////////////////////////////////////////////
58 
59  //=================================================================
60  /// The continuation timestepper object
61  //=================================================================
62  ContinuationStorageScheme Problem::Continuation_time_stepper;
63 
64  //================================================================
65  /// Constructor: Allocate space for one time stepper
66  /// and set all pointers to NULL and set defaults for all
67  /// parameters.
68  //===============================================================
70  : Mesh_pt(0),
71  Time_pt(0),
72  Explicit_time_stepper_pt(0),
73  Saved_dof_pt(0),
74  Default_set_initial_condition_called(false),
75  Use_globally_convergent_newton_method(false),
76  Empty_actions_before_read_unstructured_meshes_has_been_called(false),
77  Empty_actions_after_read_unstructured_meshes_has_been_called(false),
78  Store_local_dof_pt_in_elements(false),
79  Calculate_hessian_products_analytic(false),
80 #ifdef OOMPH_HAS_MPI
81  Doc_imbalance_in_parallel_assembly(false),
82  Use_default_partition_in_load_balance(false),
83  Must_recompute_load_balance_for_assembly(true),
84  Halo_scheme_pt(0),
85 #endif
86  Relaxation_factor(1.0),
87  Newton_solver_tolerance(1.0e-8),
89  Nnewton_iter_taken(0),
90  Max_residuals(10.0),
91  Time_adaptive_newton_crash_on_solve_fail(false),
92  Jacobian_reuse_is_enabled(false),
93  Jacobian_has_been_computed(false),
94  Problem_is_nonlinear(true),
95  Pause_at_end_of_sparse_assembly(false),
96  Doc_time_in_distribute(false),
97  Sparse_assembly_method(Perform_assembly_using_vectors_of_pairs),
98  Sparse_assemble_with_arrays_initial_allocation(400),
99  Sparse_assemble_with_arrays_allocation_increment(150),
100  Numerical_zero_for_sparse_assembly(0.0),
101  FD_step_used_in_get_hessian_vector_products(1.0e-8),
102  Mass_matrix_reuse_is_enabled(false),
103  Mass_matrix_has_been_computed(false),
104  Discontinuous_element_formulation(false),
105  Minimum_dt(1.0e-12),
106  Maximum_dt(1.0e12),
107  DTSF_max_increase(4.0),
108  DTSF_min_decrease(0.8),
109  Minimum_dt_but_still_proceed(-1.0),
110  Scale_arc_length(true),
111  Desired_proportion_of_arc_length(0.5),
112  Theta_squared(1.0),
113  Sign_of_jacobian(0),
114  Continuation_direction(1.0),
115  Parameter_derivative(1.0),
116  Parameter_current(0.0),
117  Use_continuation_timestepper(false),
118  Dof_derivative_offset(1),
119  Dof_current_offset(2),
120  Ds_current(0.0),
121  Desired_newton_iterations_ds(5),
122  Minimum_ds(1.0e-10),
123  Bifurcation_detection(false),
124  Bisect_to_find_bifurcation(false),
125  First_jacobian_sign_change(false),
126  Arc_length_step_taken(false),
127  Use_finite_differences_for_continuation_derivatives(false),
128 #ifdef OOMPH_HAS_MPI
129  Dist_problem_matrix_distribution(Uniform_matrix_distribution),
130  Parallel_sparse_assemble_previous_allocation(0),
131  Problem_has_been_distributed(false),
132  Bypass_increase_in_dof_check_during_pruning(false),
133  Max_permitted_error_for_halo_check(1.0e-14),
134 #endif
135  Shut_up_in_newton_solve(false),
136  Always_take_one_newton_step(false),
137  Timestep_reduction_factor_after_nonconvergence(0.5),
138  Keep_temporal_error_below_tolerance(true)
139  {
141 
142  /// Setup terminate helper
144 
145  // By default no submeshes:
146  Sub_mesh_pt.resize(0);
147  // No timesteppers
148  Time_stepper_pt.resize(0);
149 
150  // Set the linear solvers, eigensolver and assembly handler
153 
155 
157 
158  // setup the communicator
159 #ifdef OOMPH_HAS_MPI
161  {
163  }
164  else
165  {
167  }
168 #else
170 #endif
171 
172  // just create an empty linear algebra distribution for the
173  // DOFs
174  // this is setup when assign_eqn_numbers(...) is called.
176  }
177 
178  //================================================================
179  /// Destructor to clean up memory
180  //================================================================
182  {
183  // Delete the memory assigned for the global time
184  // (it's created on the fly in Problem::add_time_stepper_pt()
185  // so we are entitled to delete it.
186  if (Time_pt != 0)
187  {
188  delete Time_pt;
189  Time_pt = 0;
190  }
191 
192  // We're not using the default linear solver,
193  // somebody else must have built it, so that person
194  // must be in charge of killing it.
195  // We can safely delete the defaults, however
197 
200  delete Communicator_pt;
201  delete Dof_distribution_pt;
202 
203  // Delete any copies of the problem that have been created for
204  // use in adaptive bifurcation tracking.
205  // ALH: This will eventually go
206  unsigned n_copies = Copy_of_problem_pt.size();
207  for (unsigned c = 0; c < n_copies; c++)
208  {
209  delete Copy_of_problem_pt[c];
210  }
211 
212  // if this problem has sub meshes then we must delete the Mesh_pt
213  if (Sub_mesh_pt.size() != 0)
214  {
216  delete Mesh_pt;
217  }
218 
219  // Since we called the TerminateHelper setup function in the constructor,
220  // we need to delete anything that was dynamically allocated (as it's
221  // just a namespace and so doesn't have it's own destructor) in the function
223  }
224 
225  //=================================================================
226  /// Setup the count vector that records how many elements contribute
227  /// to each degree of freedom. Returns the total number of elements
228  /// in the problem
229  //=================================================================
231  {
232  // Now set the element counter to have the current Dof distribution
234  // We need to use the halo scheme (assuming it has been setup)
235 #ifdef OOMPH_HAS_MPI
237 #endif
238 
239  // Loop over the elements and count the entries
240  // and number of (non-halo) elements
241  const unsigned n_element = this->mesh_pt()->nelement();
242  unsigned n_non_halo_element_local = 0;
243  for (unsigned e = 0; e < n_element; e++)
244  {
245  GeneralisedElement* elem_pt = this->mesh_pt()->element_pt(e);
246 #ifdef OOMPH_HAS_MPI
247  // Ignore halo elements
248  if (!elem_pt->is_halo())
249  {
250 #endif
251  // Increment the number of non halo elements
252  ++n_non_halo_element_local;
253  // Now count the number of times the element contributes to a value
254  // using the current assembly handler
255  unsigned n_var = this->Assembly_handler_pt->ndof(elem_pt);
256  for (unsigned n = 0; n < n_var; n++)
257  {
259  this->Assembly_handler_pt->eqn_number(elem_pt, n));
260  }
261 #ifdef OOMPH_HAS_MPI
262  }
263 #endif
264  }
265 
266  // Storage for the total number of elements
267  unsigned Nelement = 0;
268 
269  // Add together all the counts if we are in parallel
270 #ifdef OOMPH_HAS_MPI
272 
273  // If distributed, find the total number of elements in the problem
275  {
276  // Need to gather the total number of non halo elements
277  MPI_Allreduce(&n_non_halo_element_local,
278  &Nelement,
279  1,
280  MPI_UNSIGNED,
281  MPI_SUM,
282  this->communicator_pt()->mpi_comm());
283  }
284  // Otherwise the total number is the same on each processor
285  else
286 #endif
287  {
288  Nelement = n_non_halo_element_local;
289  }
290 
291  return Nelement;
292  }
293 
294 
295  //==================================================================
296  /// Build new LinearAlgebraDistribution. Note: you're in charge of
297  /// deleting it!
298  //==================================================================
300  LinearAlgebraDistribution*& dist_pt)
301  {
302  // Find the number of rows
303  const unsigned nrow = this->ndof();
304 
305 #ifdef OOMPH_HAS_MPI
306 
307  unsigned nproc = Communicator_pt->nproc();
308 
309  // if problem is only one one processor assemble non-distributed
310  // distribution
311  if (nproc == 1)
312  {
313  dist_pt = new LinearAlgebraDistribution(Communicator_pt, nrow, false);
314  }
315  // if the problem is not distributed then assemble the jacobian with
316  // a uniform distributed distribution
318  {
319  dist_pt = new LinearAlgebraDistribution(Communicator_pt, nrow, true);
320  }
321  // otherwise the problem is a distributed problem
322  else
323  {
325  {
327 
328  dist_pt = new LinearAlgebraDistribution(Communicator_pt, nrow, true);
329  break;
330 
332 
334  break;
335 
337 
338  // Put in its own scope to avoid warnings about "local" variables
339  {
340  LinearAlgebraDistribution* uniform_dist_pt =
342  bool use_problem_dist = true;
343  for (unsigned p = 0; p < nproc; p++)
344  {
345  // hierher Andrew: what's the logic behind this?
346  if ((double)Dof_distribution_pt->nrow_local(p) >
347  ((double)uniform_dist_pt->nrow_local(p)) * 1.1)
348  {
349  use_problem_dist = false;
350  }
351  }
352  if (use_problem_dist)
353  {
355  }
356  else
357  {
358  dist_pt = new LinearAlgebraDistribution(uniform_dist_pt);
359  }
360  delete uniform_dist_pt;
361  }
362  break;
363 
364  default:
365 
366  std::ostringstream error_stream;
367  error_stream << "Never get here. Dist_problem_matrix_distribution = "
368  << Dist_problem_matrix_distribution << std::endl;
369  throw OomphLibError(error_stream.str(),
370  OOMPH_CURRENT_FUNCTION,
371  OOMPH_EXCEPTION_LOCATION);
372  break;
373  }
374  }
375 #else
376  dist_pt = new LinearAlgebraDistribution(Communicator_pt, nrow, false);
377 #endif
378  }
379 
380 
381 #ifdef OOMPH_HAS_MPI
382 
383  //==================================================================
384  /// Setup the halo scheme for the degrees of freedom
385  //==================================================================
387  {
388  // Find the number of elements stored on this processor
389  const unsigned n_element = this->mesh_pt()->nelement();
390 
391  // Work out the all global equations to which this processor
392  // contributes
393  Vector<unsigned> my_eqns;
394  this->get_my_eqns(this->Assembly_handler_pt, 0, n_element - 1, my_eqns);
395 
396  // Build the halo scheme, based on the equations to which this
397  // processor contributes
399  new DoubleVectorHaloScheme(this->Dof_distribution_pt, my_eqns);
400 
401  // Find pointers to all the halo dofs
402  // There may be more of these than required by my_eqns
403  //(but should not be less)
404  std::map<unsigned, double*> halo_data_pt;
405  this->get_all_halo_data(halo_data_pt);
406 
407  // Now setup the Halo_dofs
408  Halo_scheme_pt->setup_halo_dofs(halo_data_pt, this->Halo_dof_pt);
409  }
410 
411  //==================================================================
412  /// Distribute the problem without doc; report stats if required.
413  /// Returns actual partitioning used, e.g. for restart.
414  //==================================================================
415  Vector<unsigned> Problem::distribute(const bool& report_stats)
416  {
417  // Set dummy doc paramemters
418  DocInfo doc_info;
419  doc_info.disable_doc();
420 
421  // Set the sizes of the input and output vectors
422  unsigned n_element = mesh_pt()->nelement();
423  Vector<unsigned> element_partition(n_element, 0);
424 
425  // Distribute and return partitioning
426  return distribute(element_partition, doc_info, report_stats);
427  }
428 
429  //==================================================================
430  /// Distribute the problem according to specified partition.
431  /// If all entries in partitioning vector are zero we use METIS
432  /// to do the partitioning after all.
433  /// Returns actual partitioning used, e.g. for restart.
434  //==================================================================
436  const Vector<unsigned>& element_partition, const bool& report_stats)
437  {
438 #ifdef PARANOID
439  bool has_non_zero_entry = false;
440  unsigned n = element_partition.size();
441  for (unsigned i = 0; i < n; i++)
442  {
443  if (element_partition[i] != 0)
444  {
445  has_non_zero_entry = true;
446  break;
447  }
448  }
449  if (!has_non_zero_entry)
450  {
451  std::ostringstream warn_message;
452  warn_message << "WARNING: All entries in specified partitioning vector \n"
453  << " are zero -- will ignore this and use METIS\n"
454  << " to perform the partitioning\n";
456  warn_message.str(), "Problem::distribute()", OOMPH_EXCEPTION_LOCATION);
457  }
458 #endif
459  // Set dummy doc paramemters
460  DocInfo doc_info;
461  doc_info.disable_doc();
462 
463  // Distribute and return partitioning
464  return distribute(element_partition, doc_info, report_stats);
465  }
466 
467  //==================================================================
468  /// Distribute the problem and doc to specified DocInfo.
469  /// Returns actual partitioning used, e.g. for restart.
470  //==================================================================
472  const bool& report_stats)
473  {
474  // Set the sizes of the input and output vectors
475  unsigned n_element = mesh_pt()->nelement();
476 
477  // Dummy input vector
478  Vector<unsigned> element_partition(n_element, 0);
479 
480  // Distribute and return partitioning
481  return distribute(element_partition, doc_info, report_stats);
482  }
483 
484  //==================================================================
485  /// Distribute the problem according to specified partition.
486  /// (If all entries in partitioning vector are zero we use METIS
487  /// to do the partitioning after all) and doc.
488  /// Returns actual partitioning used, e.g. for restart.
489  //==================================================================
491  const Vector<unsigned>& element_partition,
492  DocInfo& doc_info,
493  const bool& report_stats)
494  {
495  // Storage for number of processors and number of elements in global mesh
496  int n_proc = this->communicator_pt()->nproc();
497  int my_rank = this->communicator_pt()->my_rank();
498  int n_element = mesh_pt()->nelement();
499 
500  // Vector to be returned
501  Vector<unsigned> return_element_domain;
502 
503  // Buffer extreme cases
504  if (n_proc == 1) // single-process job - don't do anything
505  {
506  if (report_stats)
507  {
508  std::ostringstream warn_message;
509  warn_message << "WARNING: You've tried to distribute a problem over\n"
510  << "only one processor: this would make METIS crash.\n"
511  << "Ignoring your request for distribution.\n";
512  OomphLibWarning(warn_message.str(),
513  "Problem::distribute()",
514  OOMPH_EXCEPTION_LOCATION);
515  }
516  }
517  else if (n_proc > n_element) // more processors than elements
518  {
519  // Throw an error
520  std::ostringstream error_stream;
521  error_stream << "You have tried to distribute a problem\n"
522  << "but there are less elements than processors.\n"
523  << "Please re-run with more elements!\n"
524  << "Please also ensure that actions_before_distribute().\n"
525  << "and actions_after_distribute() are correctly set up.\n"
526  << std::endl;
527  throw OomphLibError(
528  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
529  }
530  else
531  {
532  // We only distribute uniformly-refined meshes; buffer the case where
533  // either mesh is not uniformly refined
534  bool a_mesh_is_not_uniformly_refined = false;
535  unsigned n_mesh = nsub_mesh();
536  if (n_mesh == 0)
537  {
538  // Check refinement levels
539  if (TreeBasedRefineableMeshBase* mmesh_pt =
540  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(0)))
541  {
542  unsigned min_ref_level = 0;
543  unsigned max_ref_level = 0;
544  mmesh_pt->get_refinement_levels(min_ref_level, max_ref_level);
545  // If they are not the same
546  if (max_ref_level != min_ref_level)
547  {
548  a_mesh_is_not_uniformly_refined = true;
549  }
550  }
551  }
552  else
553  {
554  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
555  {
556  // Check refinement levels for each mesh individually
557  // (one mesh is allowed to be "more uniformly refined" than another)
558  if (TreeBasedRefineableMeshBase* mmesh_pt =
559  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(i_mesh)))
560  {
561  unsigned min_ref_level = 0;
562  unsigned max_ref_level = 0;
563  mmesh_pt->get_refinement_levels(min_ref_level, max_ref_level);
564  // If they are not the same
565  if (max_ref_level != min_ref_level)
566  {
567  a_mesh_is_not_uniformly_refined = true;
568  }
569  }
570  }
571  }
572 
573  // If any mesh is not uniformly refined
574  if (a_mesh_is_not_uniformly_refined)
575  {
576  // Again it may make more sense to throw an error here as the user
577  // will probably not be running a problem that is small enough to
578  // fit the whole of on each processor
579  std::ostringstream error_stream;
580  error_stream << "You have tried to distribute a problem\n"
581  << "but at least one of your meshes is no longer\n"
582  << "uniformly refined. In order to preserve the Tree\n"
583  << "and TreeForest structure, Problem::distribute() can\n"
584  << "only be called while meshes are uniformly refined.\n"
585  << std::endl;
586  throw OomphLibError(
587  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
588  }
589  else
590  {
591  // Is there any global data? If so, distributing the problem won't work
592  if (nglobal_data() > 0)
593  {
594  std::ostringstream error_stream;
595  error_stream << "You have tried to distribute a problem\n"
596  << "and there is some global data.\n"
597  << "This is not likely to work...\n"
598  << std::endl;
599  throw OomphLibError(error_stream.str(),
600  OOMPH_CURRENT_FUNCTION,
601  OOMPH_EXCEPTION_LOCATION);
602  }
603 
604  double t_start = 0;
606  {
607  t_start = TimingHelpers::timer();
608  }
609 
610 
611 #ifdef PARANOID
612  unsigned old_ndof = ndof();
613 #endif
614 
615  // Need to partition the global mesh before distributing
616  Mesh* global_mesh_pt = mesh_pt();
617 
618  // Vector listing the affiliation of each element
619  unsigned nelem = global_mesh_pt->nelement();
620  Vector<unsigned> element_domain(nelem);
621 
622  // Number of elements that I'm in charge of, based on any
623  // incoming partitioning
624  unsigned n_my_elements = 0;
625 
626  // Have we used the pre-set partitioning
627  bool used_preset_partitioning = false;
628 
629  // Partition the mesh, unless the partition has already been passed in
630  // If it hasn't then the sum of all the entries of the vector should be
631  // 0
632  unsigned sum_element_partition = 0;
633  unsigned n_part = element_partition.size();
634  for (unsigned e = 0; e < n_part; e++)
635  {
636  // ... another one for me.
637  if (int(element_partition[e]) == my_rank) n_my_elements++;
638 
639  sum_element_partition += element_partition[e];
640  }
641  if (sum_element_partition == 0)
642  {
643  oomph_info << "INFO: using METIS to partition elements" << std::endl;
644  partition_global_mesh(global_mesh_pt, doc_info, element_domain);
645  used_preset_partitioning = false;
646  }
647  else
648  {
649  oomph_info << "INFO: using pre-set partition of elements"
650  << std::endl;
651  used_preset_partitioning = true;
652  element_domain = element_partition;
653  }
654 
655  // Set the GLOBAL Mesh as being distributed
656  global_mesh_pt->set_communicator_pt(this->communicator_pt());
657 
658  double t_end = 0.0;
660  {
661  t_end = TimingHelpers::timer();
662  oomph_info << "Time for partitioning of global mesh: "
663  << t_end - t_start << std::endl;
664  t_start = TimingHelpers::timer();
665  }
666 
667  // Store how many elements we had in the various sub-meshes
668  // before actions_before_distribute() (which may empty some of
669  // them).
670  Vector<unsigned> n_element_in_old_submesh(n_mesh);
671  if (n_mesh != 0)
672  {
673  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
674  {
675  unsigned nsub_elem = mesh_pt(i_mesh)->nelement();
676  n_element_in_old_submesh[i_mesh] = nsub_elem;
677  }
678  }
679 
680  // Partitioning complete; call actions before distribute
682 
684  {
685  t_end = TimingHelpers::timer();
686  oomph_info << "Time for actions before distribute: "
687  << t_end - t_start << std::endl;
688  }
689 
690  // This next bit is cheap -- omit timing
691  // t_start = TimingHelpers::timer();
692 
693  // Number of submeshes (NB: some may have been deleted in
694  // actions_after_distribute())
695  n_mesh = nsub_mesh();
696 
697 
698  // Prepare vector of vectors for submesh element domains
699  Vector<Vector<unsigned>> submesh_element_domain(n_mesh);
700 
701  // The submeshes need to know their own element domains.
702  // Also if any meshes have been emptied we ignore their
703  // partitioning in the vector that we return from here
704  return_element_domain.reserve(element_domain.size());
705  if (n_mesh != 0)
706  {
707  unsigned count = 0;
708  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
709  {
710  unsigned nsub_elem = mesh_pt(i_mesh)->nelement();
711  submesh_element_domain[i_mesh].resize(nsub_elem);
712  unsigned nsub_elem_old = n_element_in_old_submesh[i_mesh];
713  for (unsigned e = 0; e < nsub_elem_old; e++)
714  {
715  if (nsub_elem_old == nsub_elem)
716  {
717  submesh_element_domain[i_mesh][e] = element_domain[count];
718  return_element_domain.push_back(element_domain[count]);
719  }
720  // return_element_domain.push_back(element_domain[count]);
721  count++;
722  }
723  }
724  }
725  else
726  {
727  return_element_domain = element_domain;
728  }
729 
731  {
732  t_start = TimingHelpers::timer();
733  }
734 
735  // Setup the map between "root" element and number in global mesh
736  // (currently used in the load_balance() routines)
737 
738  // This map is only established for structured meshes, then we
739  // need to check here the type of mesh
740  if (n_mesh == 0)
741  {
742  // Check if the only one mesh is an structured mesh
743  bool structured_mesh = true;
744  TriangleMeshBase* tri_mesh_pt =
745  dynamic_cast<TriangleMeshBase*>(mesh_pt(0));
746  if (tri_mesh_pt != 0)
747  {
748  structured_mesh = false;
749  } // if (tri_mesh_pt != 0)
750  if (structured_mesh)
751  {
752  const unsigned n_ele = global_mesh_pt->nelement();
753  Base_mesh_element_pt.resize(n_ele);
755  for (unsigned e = 0; e < n_ele; e++)
756  {
757  GeneralisedElement* el_pt = global_mesh_pt->element_pt(e);
759  Base_mesh_element_pt[e] = el_pt;
760  } // for (e<n_ele)
761  } // A TreeBaseMesh mesh
762  } // if (n_mesh==0)
763  else
764  {
765  // If we have submeshes then we only add those elements that
766  // belong to structured meshes, but first compute the number
767  // of total elements in the structured meshes
768  unsigned nglobal_element = 0;
769  // Store which submeshes are structured
770  std::vector<bool> is_structured_mesh(n_mesh);
771  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
772  {
773  TriangleMeshBase* tri_mesh_pt =
774  dynamic_cast<TriangleMeshBase*>(mesh_pt(i_mesh));
775  if (tri_mesh_pt != 0)
776  {
777  // Set the flags to indicate this is not an structured
778  // mesh
779  is_structured_mesh[i_mesh] = false;
780  } // if (tri_mesh_pt != 0)
781  else
782  {
783  // Set the flags to indicate this is an structured
784  // mesh
785  is_structured_mesh[i_mesh] = true;
786  } // else if (tri_mesh_pt != 0)
787  // Check if mesh is an structured mesh
788  if (is_structured_mesh[i_mesh])
789  {
790  nglobal_element += mesh_pt(i_mesh)->nelement();
791  } // A TreeBaseMesh mesh
792  } // for (i_mesh<n_mesh)
793 
794  // Once computed the number of elements, then resize the
795  // structure
796  Base_mesh_element_pt.resize(nglobal_element);
798  unsigned counter = 0;
799  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
800  {
801  // Check if mesh is an structured mesh
802  if (is_structured_mesh[i_mesh])
803  {
804  const unsigned n_ele = mesh_pt(i_mesh)->nelement();
805  for (unsigned e = 0; e < n_ele; e++)
806  {
807  GeneralisedElement* el_pt = mesh_pt(i_mesh)->element_pt(e);
808  Base_mesh_element_number_plus_one[el_pt] = counter + 1;
809  Base_mesh_element_pt[counter] = el_pt;
810  // Inrease the global element number
811  counter++;
812  } // for (e<n_ele)
813  } // An structured mesh
814  } // for (i_mesh<n_mesh)
815 
816 #ifdef PARANOID
817  if (counter != nglobal_element)
818  {
819  std::ostringstream error_stream;
820  error_stream
821  << "The number of global elements (" << nglobal_element
822  << ") is not the sameas the number of\nadded elements ("
823  << counter << ") to the Base_mesh_element_pt data "
824  << "structure!!!\n\n";
825  throw OomphLibError(error_stream.str(),
826  "Problem::distribute()",
827  OOMPH_EXCEPTION_LOCATION);
828  } // if (counter != nglobal_element)
829 #endif // #ifdef PARANOID
830 
831  } // else if (n_mesh==0)
832 
833  // Wipe everything if a pre-determined partitioning
834  // didn't specify ANY elements for this processor
835  // (typically happens during restarts with larger number
836  // of processors -- in this case we really want an empty
837  // processor rather than one with any "kept" halo elements)
838  bool overrule_keep_as_halo_element_status = false;
839  if ((n_my_elements == 0) && (used_preset_partitioning))
840  {
841  oomph_info << "INFO: We're over-ruling the \"keep as halo element\"\n"
842  << " status because the preset partitioning\n"
843  << " didn't place ANY elements on this processor,\n"
844  << " probably because of a restart on a larger \n"
845  << " number of processors\n";
846  overrule_keep_as_halo_element_status = true;
847  }
848 
849 
850  // Distribute the (sub)meshes (i.e. sort out their halo lookup schemes)
851  Vector<GeneralisedElement*> deleted_element_pt;
852  if (n_mesh == 0)
853  {
854  global_mesh_pt->distribute(this->communicator_pt(),
855  element_domain,
856  deleted_element_pt,
857  doc_info,
858  report_stats,
859  overrule_keep_as_halo_element_status);
860  }
861  else // There are submeshes, "distribute" each one separately
862  {
863  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
864  {
865  if (report_stats)
866  {
867  oomph_info << "Distributing submesh " << i_mesh << std::endl
868  << "--------------------" << std::endl;
869  }
870  // Set the doc_info number to reflect the submesh
871  doc_info.number() = i_mesh;
872  mesh_pt(i_mesh)->distribute(this->communicator_pt(),
873  submesh_element_domain[i_mesh],
874  deleted_element_pt,
875  doc_info,
876  report_stats,
877  overrule_keep_as_halo_element_status);
878  }
879  // Rebuild the global mesh
881  }
882 
883  // Null out information associated with deleted elements
884  unsigned n_del = deleted_element_pt.size();
885  for (unsigned e = 0; e < n_del; e++)
886  {
887  GeneralisedElement* el_pt = deleted_element_pt[e];
888  unsigned old_el_number = Base_mesh_element_number_plus_one[el_pt] - 1;
890  Base_mesh_element_pt[old_el_number] = 0;
891  }
892 
894  {
895  t_end = TimingHelpers::timer();
896  oomph_info << "Time for mesh-level distribution: " << t_end - t_start
897  << std::endl;
898  t_start = TimingHelpers::timer();
899  }
900 
901  // Now the problem has been distributed
903 
904  // Call actions after distribute
906 
908  {
909  t_end = TimingHelpers::timer();
910  oomph_info << "Time for actions after distribute: " << t_end - t_start
911  << std::endl;
912  t_start = TimingHelpers::timer();
913  }
914 
915  // Re-assign the equation numbers (incl synchronisation if reqd)
916  unsigned n_dof = assign_eqn_numbers();
917  oomph_info << "Number of equations: " << n_dof << std::endl;
918 
920  {
921  t_end = TimingHelpers::timer();
922  oomph_info << "Time for re-assigning eqn numbers (in distribute): "
923  << t_end - t_start << std::endl;
924  }
925 
926 
927 #ifdef PARANOID
928  if (n_dof != old_ndof)
929  {
930  std::ostringstream error_stream;
931  error_stream
932  << "Number of dofs in distribute() has changed "
933  << "from " << old_ndof << " to " << n_dof << "\n"
934  << "Check that you've implemented any necessary "
935  "actions_before/after\n"
936  << "distribute functions, e.g. to pin redundant pressure dofs"
937  << " etc.\n";
938  throw OomphLibError(error_stream.str(),
939  OOMPH_CURRENT_FUNCTION,
940  OOMPH_EXCEPTION_LOCATION);
941  }
942 #endif
943 
944  } // end if to check for uniformly refined mesh(es)
945 
946  } // end if to check number of processors vs. number of elements etc.
947 
948 
949  // Force re-analysis of time spent on assembly each
950  // elemental Jacobian
952  Elemental_assembly_time.clear();
953 
954  // Return the partition vector used in the distribution
955  return return_element_domain;
956  }
957 
958  //==================================================================
959  /// Partition the global mesh, return vector specifying the processor
960  /// number for each element. Virtual so that it can be overloaded by
961  /// any user; the default is to use METIS to perform the partitioning
962  /// (with a bit of cleaning up afterwards to sort out "special cases").
963  //==================================================================
964  void Problem::partition_global_mesh(Mesh*& global_mesh_pt,
965  DocInfo& doc_info,
966  Vector<unsigned>& element_domain,
967  const bool& report_stats)
968  {
969  // Storage for number of processors and current processor
970  int n_proc = this->communicator_pt()->nproc();
971  int rank = this->communicator_pt()->my_rank();
972 
973  std::ostringstream filename;
974  std::ofstream some_file;
975 
976  // Doc the original mesh on proc 0
977  //--------------------------------
978  if (doc_info.is_doc_enabled())
979  {
980  if (rank == 0)
981  {
982  filename << doc_info.directory() << "/complete_mesh"
983  << doc_info.number() << ".dat";
984  global_mesh_pt->output(filename.str().c_str(), 5);
985  }
986  }
987 
988  // Partition the mesh
989  //-------------------
990  // METIS Objective (0: minimise edge cut; 1: minimise total comm volume)
991  unsigned objective = 0;
992 
993  // Do the partitioning
994  unsigned nelem = 0;
995  if (this->communicator_pt()->my_rank() == 0)
996  {
997  METIS::partition_mesh(this, n_proc, objective, element_domain);
998  nelem = element_domain.size();
999  }
1000  MPI_Bcast(&nelem, 1, MPI_UNSIGNED, 0, this->communicator_pt()->mpi_comm());
1001  element_domain.resize(nelem);
1002  MPI_Bcast(&element_domain[0],
1003  nelem,
1004  MPI_UNSIGNED,
1005  0,
1006  this->communicator_pt()->mpi_comm());
1007 
1008  // On very coarse meshes with larger numbers of processors, METIS
1009  // occasionally returns an element_domain Vector for which a particular
1010  // processor has no elements affiliated to it; the following fixes this
1011 
1012  // Convert element_domain to integer storage
1013  Vector<int> int_element_domain(nelem);
1014  for (unsigned e = 0; e < nelem; e++)
1015  {
1016  int_element_domain[e] = element_domain[e];
1017  }
1018 
1019  // Global storage for number of elements on each process
1020  int my_number_of_elements = 0;
1021  Vector<int> number_of_elements(n_proc, 0);
1022 
1023  for (unsigned e = 0; e < nelem; e++)
1024  {
1025  if (int_element_domain[e] == rank)
1026  {
1027  my_number_of_elements++;
1028  }
1029  }
1030 
1031  // Communicate the correct value for each single process into
1032  // the global storage vector
1033  MPI_Allgather(&my_number_of_elements,
1034  1,
1035  MPI_INT,
1036  &number_of_elements[0],
1037  1,
1038  MPI_INT,
1039  this->communicator_pt()->mpi_comm());
1040 
1041  // If a process has no elements then switch an element with the
1042  // process with the largest number of elements, assuming
1043  // that it still has enough elements left to share
1044  int max_number_of_elements = 0;
1045  int process_with_max_elements = 0;
1046  for (int d = 0; d < n_proc; d++)
1047  {
1048  if (number_of_elements[d] == 0)
1049  {
1050  // Find the process with maximum number of elements
1051  if (max_number_of_elements <= 1)
1052  {
1053  for (int dd = 0; dd < n_proc; dd++)
1054  {
1055  if (number_of_elements[dd] > max_number_of_elements)
1056  {
1057  max_number_of_elements = number_of_elements[dd];
1058  process_with_max_elements = dd;
1059  }
1060  }
1061  }
1062 
1063  // Check that this number of elements is okay for sharing
1064  if (max_number_of_elements <= 1)
1065  {
1066  // Throw an error if elements can't be shared
1067  std::ostringstream error_stream;
1068  error_stream << "No process has more than 1 element, and\n"
1069  << "at least one process has no elements!\n"
1070  << "Suggest rerunning with more refinement.\n"
1071  << std::endl;
1072  throw OomphLibError(error_stream.str(),
1073  OOMPH_CURRENT_FUNCTION,
1074  OOMPH_EXCEPTION_LOCATION);
1075  }
1076 
1077  // Loop over the element domain vector and switch
1078  // one value for process "process_with_max_elements" with d
1079  for (unsigned e = 0; e < nelem; e++)
1080  {
1081  if (int_element_domain[e] == process_with_max_elements)
1082  {
1083  int_element_domain[e] = d;
1084  // Change the numbers associated with these processes
1085  number_of_elements[d]++;
1086  number_of_elements[process_with_max_elements]--;
1087  // Reduce the number of elements available on "max" process
1088  max_number_of_elements--;
1089  // Inform the user that a switch has taken place
1090  if (report_stats)
1091  {
1092  oomph_info << "INFO: Switched element domain at position " << e
1093  << std::endl
1094  << "from process " << process_with_max_elements
1095  << " to process " << d << std::endl
1096  << "which was given no elements by METIS partition"
1097  << std::endl;
1098  }
1099  // Only need to do this once for this element loop, otherwise
1100  // this will take all the elements from "max" process and put them
1101  // in process d, thus leaving essentially the same problem!
1102  break;
1103  }
1104  }
1105  }
1106  }
1107 
1108  // Reassign new values to the element_domain vector
1109  for (unsigned e = 0; e < nelem; e++)
1110  {
1111  element_domain[e] = int_element_domain[e];
1112  }
1113 
1114  unsigned count_elements = 0;
1115  for (unsigned e = 0; e < nelem; e++)
1116  {
1117  if (int(element_domain[e]) == rank)
1118  {
1119  count_elements++;
1120  }
1121  }
1122 
1123  if (report_stats)
1124  {
1125  oomph_info << "I have " << count_elements
1126  << " elements from this partition" << std::endl
1127  << std::endl;
1128  }
1129  }
1130 
1131  //==================================================================
1132  /// (Irreversibly) prune halo(ed) elements and nodes, usually
1133  /// after another round of refinement, to get rid of
1134  /// excessively wide halo layers. Note that the current
1135  /// mesh will be now regarded as the base mesh and no unrefinement
1136  /// relative to it will be possible once this function
1137  /// has been called.
1138  //==================================================================
1140  const bool& report_stats)
1141  {
1142  // Storage for number of processors and current processor
1143  int n_proc = this->communicator_pt()->nproc();
1144 
1145  // Has the problem been distributed yet?
1147  {
1148  oomph_info
1149  << "WARNING: Problem::prune_halo_elements_and_nodes() was called on a "
1150  << "non-distributed Problem!" << std::endl;
1151  oomph_info << "Ignoring your request..." << std::endl;
1152  }
1153  else
1154  {
1155  // There are no halo layers to prune if it's a single-process job
1156  if (n_proc == 1)
1157  {
1158  oomph_info
1159  << "WARNING: You've tried to prune halo layers on a problem\n"
1160  << "with only one processor: this is unnecessary.\n"
1161  << "Ignoring your request." << std::endl
1162  << std::endl;
1163  }
1164  else
1165  {
1166 #ifdef PARANOID
1167  unsigned old_ndof = ndof();
1168 #endif
1169 
1170  double t_start = 0.0;
1172  {
1173  t_start = TimingHelpers::timer();
1174  }
1175 
1176  // Call actions before distribute
1178 
1179  double t_end = 0.0;
1181  {
1182  t_end = TimingHelpers::timer();
1183  oomph_info << "Time for actions_before_distribute() in "
1184  << "Problem::prune_halo_elements_and_nodes(): "
1185  << t_end - t_start << std::endl;
1186  t_start = TimingHelpers::timer();
1187  }
1188 
1189  // Associate all elements with root in current Base mesh
1190  unsigned nel = Base_mesh_element_pt.size();
1191  std::map<GeneralisedElement*, unsigned>
1192  old_base_element_number_plus_one;
1193  std::vector<bool> old_root_is_halo_or_non_existent(nel, true);
1194  for (unsigned e = 0; e < nel; e++)
1195  {
1196  // Get the base element
1197  GeneralisedElement* base_el_pt = Base_mesh_element_pt[e];
1198 
1199  // Does it exist locally?
1200  if (base_el_pt != 0)
1201  {
1202  // Check if it's a halo element
1203  if (!base_el_pt->is_halo())
1204  {
1205  old_root_is_halo_or_non_existent[e] = false;
1206  }
1207 
1208  // Not refineable: It's only the element iself
1209  RefineableElement* ref_el_pt = 0;
1210  ref_el_pt = dynamic_cast<RefineableElement*>(base_el_pt);
1211  if (ref_el_pt == 0)
1212  {
1213  old_base_element_number_plus_one[base_el_pt] = e + 1;
1214  }
1215  // Refineable: Get entire tree of elements
1216  else
1217  {
1218  Vector<Tree*> tree_pt;
1219  ref_el_pt->tree_pt()->stick_all_tree_nodes_into_vector(tree_pt);
1220  unsigned ntree = tree_pt.size();
1221  for (unsigned t = 0; t < ntree; t++)
1222  {
1223  old_base_element_number_plus_one[tree_pt[t]->object_pt()] =
1224  e + 1;
1225  }
1226  }
1227  }
1228  }
1229 
1230 
1232  {
1233  t_end = TimingHelpers::timer();
1234  oomph_info << "Time for setup old root elements in "
1235  << "Problem::prune_halo_elements_and_nodes(): "
1236  << t_end - t_start << std::endl;
1237  t_start = TimingHelpers::timer();
1238  }
1239 
1240 
1241  // Now remember the old number of base elements
1242  unsigned nel_base_old = nel;
1243 
1244 
1245  // Prune the halo elements and nodes of the mesh(es)
1246  Vector<GeneralisedElement*> deleted_element_pt;
1247  unsigned n_mesh = nsub_mesh();
1248  if (n_mesh == 0)
1249  {
1250  // Prune halo elements and nodes for the (single) global mesh
1252  deleted_element_pt, doc_info, report_stats);
1253  }
1254  else
1255  {
1256  // Loop over individual submeshes and prune separately
1257  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
1258  {
1260  deleted_element_pt, doc_info, report_stats);
1261  }
1262 
1263  // Rebuild the global mesh
1265  }
1266 
1268  {
1269  t_end = TimingHelpers::timer();
1270  oomph_info << "Total time for all mesh-level prunes in "
1271  << "Problem::prune_halo_elements_and_nodes(): "
1272  << t_end - t_start << std::endl;
1273  t_start = TimingHelpers::timer();
1274  }
1275 
1276  // Loop over all elements in newly rebuilt mesh (which contains
1277  // all element in "tree order"), find the roots
1278  // (which are either non-refineable elements or refineable elements
1279  // whose tree representations are TreeRoots)
1280  std::map<FiniteElement*, bool> root_el_done;
1281 
1282  // Vector storing vectors of pointers to new base elements associated
1283  // with the same old base element
1285  new_base_element_associated_with_old_base_element(nel_base_old);
1286 
1287  unsigned n_meshes = n_mesh;
1288  // Change the value for the number of submeshes if there is only
1289  // one mesh so that the loop below works if we have only one
1290  // mesh
1291  if (n_meshes == 0)
1292  {
1293  n_meshes = 1;
1294  }
1295 
1296  // Store which submeshes, if there are some are structured
1297  // meshes
1298  std::vector<bool> is_structured_mesh(n_meshes);
1299 
1300  // Loop over all elements in the rebuilt mesh, but make sure
1301  // that we are only looping over the structured meshes
1302  nel = 0;
1303  for (unsigned i_mesh = 0; i_mesh < n_meshes; i_mesh++)
1304  {
1305  TriangleMeshBase* tri_mesh_pt =
1306  dynamic_cast<TriangleMeshBase*>(mesh_pt(i_mesh));
1307  if (!(tri_mesh_pt != 0))
1308  {
1309  // Mark the mesh as structured mesh
1310  is_structured_mesh[i_mesh] = true;
1311  // Add the number of elements
1312  nel += mesh_pt(i_mesh)->nelement();
1313  } // if (!(tri_mesh_pt!=0))
1314  else
1315  {
1316  // Mark the mesh as nonstructured mesh
1317  is_structured_mesh[i_mesh] = false;
1318  } // else if (!(tri_mesh_pt!=0))
1319  } // for (i_mesh < n_mesh)
1320 
1321  // Go for all the meshes (if there are submeshes)
1322  for (unsigned i_mesh = 0; i_mesh < n_meshes; i_mesh++)
1323  {
1324  // Only work with the elements in the mesh if it is an
1325  // structured mesh
1326  if (is_structured_mesh[i_mesh])
1327  {
1328  // Get the number of elements in the submesh
1329  const unsigned nele_submesh = mesh_pt(i_mesh)->nelement();
1330  for (unsigned e = 0; e < nele_submesh; e++)
1331  {
1332  // Get the element
1333  GeneralisedElement* el_pt = mesh_pt(i_mesh)->element_pt(e);
1334 
1335  // Not refineable: It's definitely a new base element
1336  RefineableElement* ref_el_pt = 0;
1337  ref_el_pt = dynamic_cast<RefineableElement*>(el_pt);
1338  if (ref_el_pt == 0)
1339  {
1340  unsigned old_base_el_no =
1341  old_base_element_number_plus_one[el_pt] - 1;
1342  new_base_element_associated_with_old_base_element
1343  [old_base_el_no]
1344  .push_back(el_pt);
1345  }
1346  // Refineable
1347  else
1348  {
1349  // Is it a tree root (after pruning)? In that case it's
1350  // a new base element
1351  if (dynamic_cast<TreeRoot*>(ref_el_pt->tree_pt()))
1352  {
1353  unsigned old_base_el_no =
1354  old_base_element_number_plus_one[el_pt] - 1;
1355  new_base_element_associated_with_old_base_element
1356  [old_base_el_no]
1357  .push_back(el_pt);
1358  }
1359  else
1360  {
1361  // Get associated root element
1362  FiniteElement* root_el_pt =
1363  ref_el_pt->tree_pt()->root_pt()->object_pt();
1364 
1365  if (!root_el_done[root_el_pt])
1366  {
1367  root_el_done[root_el_pt] = true;
1368  unsigned old_base_el_no =
1369  old_base_element_number_plus_one[el_pt] - 1;
1370  new_base_element_associated_with_old_base_element
1371  [old_base_el_no]
1372  .push_back(root_el_pt);
1373  }
1374  }
1375  }
1376  } // for (e < nele_submesh)
1377  } // if (is_structured_mesh[i_mesh])
1378  } // for (i_mesh < n_mesh)
1379 
1380  // Create a vector that stores how many new root/base elements
1381  // got spawned from each old root/base element in the global mesh
1382  Vector<unsigned> local_n_new_root(nel_base_old);
1383 #ifdef PARANOID
1384  Vector<unsigned> n_new_root_back(nel_base_old);
1385 #endif
1386  for (unsigned e = 0; e < nel_base_old; e++)
1387  {
1388  local_n_new_root[e] =
1389  new_base_element_associated_with_old_base_element[e].size();
1390 
1391 #ifdef PARANOID
1392  // Backup so we can check that halo data was consistent
1393  n_new_root_back[e] = local_n_new_root[e];
1394 #endif
1395  }
1396 
1398  {
1399  t_end = TimingHelpers::timer();
1400  oomph_info << "Time for setup of new base elements in "
1401  << "Problem::prune_halo_elements_and_nodes(): "
1402  << t_end - t_start << std::endl;
1403  t_start = TimingHelpers::timer();
1404  }
1405 
1406  // Now do reduce operation to get information for all
1407  // old root/base elements -- the pruned (halo!) base elements contain
1408  // fewer associated new roots.
1409  Vector<unsigned> n_new_root(nel_base_old);
1410  MPI_Allreduce(&local_n_new_root[0],
1411  &n_new_root[0],
1412  nel_base_old,
1413  MPI_UNSIGNED,
1414  MPI_MAX,
1415  this->communicator_pt()->mpi_comm());
1416 
1417 
1419  {
1420  t_end = TimingHelpers::timer();
1421  oomph_info << "Time for allreduce in "
1422  << "Problem::prune_halo_elements_and_nodes(): "
1423  << t_end - t_start << std::endl;
1424  t_start = TimingHelpers::timer();
1425  }
1426 
1427  // Find out total number of base elements
1428  unsigned nel_base_new = 0;
1429  for (unsigned e = 0; e < nel_base_old; e++)
1430  {
1431  // Increment
1432  nel_base_new += n_new_root[e];
1433 
1434 #ifdef PARANOID
1435  // If we already had data for this root previously then
1436  // the data ought to be consistent afterwards (since taking
1437  // the max of consistent numbers shouldn't change things -- this
1438  // deals with halo/haloed elements)
1439  if (!old_root_is_halo_or_non_existent[e])
1440  {
1441  if (n_new_root_back[e] != 0)
1442  {
1443  if (n_new_root_back[e] != n_new_root[e])
1444  {
1445  std::ostringstream error_stream;
1446  error_stream
1447  << "Number of new root elements spawned from old root " << e
1448  << ": " << n_new_root[e] << "\nis not consistent"
1449  << " with previous value: " << n_new_root_back[e]
1450  << std::endl;
1451  throw OomphLibError(error_stream.str(),
1452  OOMPH_CURRENT_FUNCTION,
1453  OOMPH_EXCEPTION_LOCATION);
1454  }
1455  }
1456  }
1457 
1458 #endif
1459  }
1460 
1461  // Reset base_mesh information
1462  Base_mesh_element_pt.clear();
1463  Base_mesh_element_pt.resize(nel_base_new, 0);
1465 
1466  // Now enumerate the new base/root elements consistently
1467  unsigned count = 0;
1468  for (unsigned e = 0; e < nel_base_old; e++)
1469  {
1470  // Old root is non-halo: Just add the new roots into the
1471  // new lookup scheme consecutively
1472  if (!old_root_is_halo_or_non_existent[e])
1473  {
1474  // Loop over new root/base element
1475  unsigned n_new_root =
1476  new_base_element_associated_with_old_base_element[e].size();
1477  for (unsigned j = 0; j < n_new_root; j++)
1478  {
1479  // Store new root/base element
1480  GeneralisedElement* el_pt =
1481  new_base_element_associated_with_old_base_element[e][j];
1482  Base_mesh_element_pt[count] = el_pt;
1483  Base_mesh_element_number_plus_one[el_pt] = count + 1;
1484 
1485  // Bump counter
1486  count++;
1487  }
1488  }
1489  // Old root element is halo so skip insertion (i.e. leave
1490  // entries in lookup schemes nulled) but increase counter to
1491  // ensure consistency between processors
1492  else
1493  {
1494  unsigned nskip = n_new_root[e];
1495  count += nskip;
1496  }
1497  }
1498 
1499  // Re-setup the map between "root" element and number in global mesh
1500  // (used in the load_balance() routines)
1502 
1503 
1505  {
1506  t_end = TimingHelpers::timer();
1507  oomph_info << "Time for finishing off base mesh info "
1508  << "Problem::prune_halo_elements_and_nodes(): "
1509  << t_end - t_start << std::endl;
1510  t_start = TimingHelpers::timer();
1511  }
1512 
1513 
1514  // Call actions after distribute
1516 
1517 
1519  {
1520  t_end = TimingHelpers::timer();
1521  oomph_info << "Time for actions_after_distribute() "
1522  << "Problem::prune_halo_elements_and_nodes(): "
1523  << t_end - t_start << std::endl;
1524  t_start = TimingHelpers::timer();
1525  }
1526 
1527 
1528  // Re-assign the equation numbers (incl synchronisation if reqd)
1529 #ifdef PARANOID
1530  unsigned n_dof = assign_eqn_numbers();
1531 #else
1533 #endif
1534 
1535 
1537  {
1538  t_end = TimingHelpers::timer();
1539  oomph_info << "Time for assign_eqn_numbers() "
1540  << "Problem::prune_halo_elements_and_nodes(): "
1541  << t_end - t_start << std::endl;
1542  t_start = TimingHelpers::timer();
1543  }
1544 
1545 
1546 #ifdef PARANOID
1548  {
1549  if (n_dof != old_ndof)
1550  {
1551  std::ostringstream error_stream;
1552  error_stream
1553  << "Number of dofs in prune_halo_elements_and_nodes() has "
1554  "changed "
1555  << "from " << old_ndof << " to " << n_dof << "\n"
1556  << "Check that you've implemented any necessary "
1557  "actions_before/after"
1558  << "\nadapt/distribute functions, e.g. to pin redundant pressure"
1559  << " dofs etc.\n";
1560  throw OomphLibError(error_stream.str(),
1561  OOMPH_CURRENT_FUNCTION,
1562  OOMPH_EXCEPTION_LOCATION);
1563  }
1564  }
1565 #endif
1566  }
1567  }
1568  }
1569 
1570 
1571 #endif
1572 
1573 
1574  //===================================================================
1575  /// Build a single (global) mesh from a number
1576  /// of submeshes which are passed as a vector of pointers to the
1577  /// submeshes. The ordering is not necessarily optimal.
1578  //==============================================================
1580  {
1581 #ifdef PARANOID
1582  // Has a global mesh already been built
1583  if (Mesh_pt != 0)
1584  {
1585  std::string error_message = "Problem::build_global_mesh() called,\n";
1586  error_message += " but a global mesh has already been built:\n";
1587  error_message += "Problem::Mesh_pt is not zero!\n";
1588 
1589  throw OomphLibError(
1590  error_message, OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
1591  }
1592  // Check that there are submeshes
1593  if (Sub_mesh_pt.size() == 0)
1594  {
1595  std::string error_message = "Problem::build_global_mesh() called,\n";
1596  error_message += " but there are no submeshes:\n";
1597  error_message += "Problem::Sub_mesh_pt has no entries\n";
1598 
1599  throw OomphLibError(
1600  error_message, OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
1601  }
1602 #endif
1603 
1604  // Create an empty mesh
1605  Mesh_pt = new Mesh();
1606 
1607  // Call the rebuild function to construct the mesh
1609  }
1610 
1611  //====================================================================
1612  /// If one of the submeshes has changed (e.g. by
1613  /// mesh adaptation) we need to update the global mesh.
1614  /// \b Note: The nodes boundary information refers to the
1615  /// boundary numbers within the submesh!
1616  /// N.B. This is essentially the same function as the Mesh constructor
1617  /// that assembles a single global mesh from submeshes
1618  //=====================================================================
1620  {
1621  // Use the function in mesh to merge the submeshes into this one
1623  }
1624 
1625 
1626  //================================================================
1627  /// Add a timestepper to the problem. The function will automatically
1628  /// create or resize the Time object so that it contains the appropriate
1629  /// number of levels of storage.
1630  //================================================================
1631  void Problem::add_time_stepper_pt(TimeStepper* const& time_stepper_pt)
1632  {
1633  // Add the timestepper to the vector
1634  Time_stepper_pt.push_back(time_stepper_pt);
1635 
1636  // Find the number of timesteps required by the timestepper
1637  unsigned ndt = time_stepper_pt->ndt();
1638 
1639  // If time has not been allocated, create time object with the
1640  // required number of time steps
1641  if (Time_pt == 0)
1642  {
1643  Time_pt = new Time(ndt);
1644  oomph_info << "Created Time with " << ndt << " timesteps" << std::endl;
1645  }
1646  else
1647  {
1648  // If the required number of time steps is greater than currently stored
1649  // resize the time storage
1650  if (ndt > Time_pt->ndt())
1651  {
1652  Time_pt->resize(ndt);
1653  oomph_info << "Resized Time to include " << ndt << " timesteps"
1654  << std::endl;
1655  }
1656  // Otherwise report that we are OK
1657  else
1658  {
1659  oomph_info << "Time object already has storage for " << ndt
1660  << " timesteps" << std::endl;
1661  }
1662  }
1663 
1664  // Pass the pointer to time to the timestepper
1666  }
1667 
1668  //================================================================
1669  /// Set the explicit time stepper for the problem and also
1670  /// ensure that a time object has been created.
1671  //================================================================
1673  ExplicitTimeStepper* const& explicit_time_stepper_pt)
1674  {
1675  // Set the explicit time stepper
1677 
1678  // If time has not been allocated, create time object with the
1679  // required number of time steps
1680  if (Time_pt == 0)
1681  {
1682  Time_pt = new Time(0);
1683  oomph_info << "Created Time with storage for no previous timestep"
1684  << std::endl;
1685  }
1686  else
1687  {
1688  oomph_info << "Time object already exists " << std::endl;
1689  }
1690  }
1691 
1692 
1693 #ifdef OOMPH_HAS_MPI
1694 
1695  //================================================================
1696  /// Set default first and last elements for parallel assembly
1697  /// of non-distributed problem.
1698  //================================================================
1700  {
1702  {
1703  // Minimum number of elements per processor if there are fewer elements
1704  // than processors
1705  unsigned min_el = 10;
1706 
1707  // Resize and make default assignments
1708  int n_proc = this->communicator_pt()->nproc();
1709  unsigned n_elements = Mesh_pt->nelement();
1710  First_el_for_assembly.resize(n_proc, 0);
1711  Last_el_plus_one_for_assembly.resize(n_proc, 0);
1712 
1713  // In the absence of any better knowledge distribute work evenly
1714  // over elements
1715  unsigned range = 0;
1716  unsigned lo_proc = 0;
1717  unsigned hi_proc = n_proc - 1;
1718  if (int(n_elements) >= n_proc)
1719  {
1720  range = unsigned(double(n_elements) / double(n_proc));
1721  }
1722  else
1723  {
1724  range = min_el;
1725  lo_proc = 0;
1726  hi_proc = unsigned(double(n_elements) / double(min_el));
1727  }
1728 
1729  for (int p = lo_proc; p <= int(hi_proc); p++)
1730  {
1731  First_el_for_assembly[p] = p * range;
1732 
1733  unsigned last_el_plus_one = (p + 1) * range;
1734  if (last_el_plus_one > n_elements) last_el_plus_one = n_elements;
1735  Last_el_plus_one_for_assembly[p] = last_el_plus_one;
1736  }
1737 
1738  // Last one needs to incorporate any dangling elements
1739  if (int(n_elements) >= n_proc)
1740  {
1741  Last_el_plus_one_for_assembly[n_proc - 1] = n_elements;
1742  }
1743 
1744  // Doc
1745  if (n_proc > 1)
1746  {
1748  {
1749  oomph_info << "Problem is not distributed. Parallel assembly of "
1750  << "Jacobian uses default partitioning: " << std::endl;
1751  for (int p = 0; p < n_proc; p++)
1752  {
1753  if (Last_el_plus_one_for_assembly[p] != 0)
1754  {
1755  oomph_info << "Proc " << p << " assembles from element "
1756  << First_el_for_assembly[p] << " to "
1757  << Last_el_plus_one_for_assembly[p] - 1 << " \n";
1758  }
1759  else
1760  {
1761  oomph_info << "Proc " << p << " assembles no elements\n";
1762  }
1763  }
1764  }
1765  }
1766  }
1767  }
1768 
1769 
1770  //=======================================================================
1771  /// Helper function to re-assign the first and last elements to be
1772  /// assembled by each processor during parallel assembly for
1773  /// non-distributed problem.
1774  //=======================================================================
1776  {
1777  // Wait until all processes have completed/timed their assembly
1778  MPI_Barrier(this->communicator_pt()->mpi_comm());
1779 
1780  // Storage for number of processors and current processor
1781  int n_proc = this->communicator_pt()->nproc();
1782  int rank = this->communicator_pt()->my_rank();
1783 
1784  // Don't bother to update if we've got fewer elements than
1785  // processors
1786  unsigned nel = Elemental_assembly_time.size();
1787  if (int(nel) < n_proc)
1788  {
1789  oomph_info << "Not re-computing distribution of elemental assembly\n"
1790  << "because there are fewer elements than processors\n";
1791  return;
1792  }
1793 
1794  // Setup vectors storing the number of element timings to be sent
1795  // and the offset in the final vector
1796  Vector<int> receive_count(n_proc);
1797  Vector<int> displacement(n_proc);
1798  int offset = 0;
1799  for (int p = 0; p < n_proc; p++)
1800  {
1801  // Default distribution of labour
1802  unsigned el_lo = First_el_for_assembly[p];
1803  unsigned el_hi = Last_el_plus_one_for_assembly[p] - 1;
1804 
1805  // Number of timings to be sent and offset from start in
1806  // final vector
1807  receive_count[p] = el_hi - el_lo + 1;
1808  displacement[p] = offset;
1809  offset += el_hi - el_lo + 1;
1810  }
1811 
1812  // Make temporary c-style array to avoid over-writing in Gatherv below
1813  double* el_ass_time = new double[nel];
1814  for (unsigned e = 0; e < nel; e++)
1815  {
1816  el_ass_time[e] = Elemental_assembly_time[e];
1817  }
1818 
1819  // Gather timings on root processor
1820  unsigned nel_local =
1822  MPI_Gatherv(&el_ass_time[First_el_for_assembly[rank]],
1823  nel_local,
1824  MPI_DOUBLE,
1826  &receive_count[0],
1827  &displacement[0],
1828  MPI_DOUBLE,
1829  0,
1830  this->communicator_pt()->mpi_comm());
1831  delete[] el_ass_time;
1832 
1833  // Vector of first and last elements for each processor
1834  Vector<Vector<int>> first_and_last_element(n_proc);
1835  for (int p = 0; p < n_proc; p++)
1836  {
1837  first_and_last_element[p].resize(2);
1838  }
1839 
1840  // Re-distribute work
1841  if (rank == 0)
1842  {
1844  {
1845  oomph_info
1846  << std::endl
1847  << "Re-assigning distribution of element assembly over processors:"
1848  << std::endl;
1849  }
1850 
1851  // Get total assembly time
1852  double total = 0.0;
1853  unsigned n_elements = Mesh_pt->nelement();
1854  for (unsigned e = 0; e < n_elements; e++)
1855  {
1856  total += Elemental_assembly_time[e];
1857  }
1858 
1859  // Target load per processor
1860  double target_load = total / double(n_proc);
1861 
1862  // We're on the root processor: Always start with the first element
1863  int proc = 0;
1864  first_and_last_element[0][0] = 0;
1865 
1866  // Highest element we can help ourselves to if we want to leave
1867  // at least one element for all subsequent processors
1868  unsigned max_el_avail = n_elements - n_proc;
1869 
1870  // Initialise total work allocated
1871  total = 0.0;
1872  for (unsigned e = 0; e < n_elements; e++)
1873  {
1874  total += Elemental_assembly_time[e];
1875 
1876  // Once we have reached the target load or we've used up practically
1877  // all the elements...
1878  if ((total > target_load) || (e == max_el_avail))
1879  {
1880  // Last element for current processor
1881  first_and_last_element[proc][1] = e;
1882 
1883  // Provided that we are not on the last processor
1884  if (proc < (n_proc - 1))
1885  {
1886  // Set first element for next one
1887  first_and_last_element[proc + 1][0] = e + 1;
1888 
1889  // Move on to the next processor
1890  proc++;
1891  }
1892 
1893  // Can have one more...
1894  max_el_avail++;
1895 
1896  // Re-initialise the time
1897  total = 0.0;
1898  } // end of test for "total exceeds target"
1899  }
1900 
1901 
1902  // Last element for last processor
1903  first_and_last_element[n_proc - 1][1] = n_elements - 1;
1904 
1905 
1906  // The following block should probably be paranoidified away
1907  // but we've screwed the logic up so many times that I feel
1908  // it's safer to keep it...
1909  bool wrong = false;
1910  std::ostringstream error_stream;
1911  for (int p = 0; p < n_proc - 1; p++)
1912  {
1913  unsigned first_of_current = first_and_last_element[p][0];
1914  unsigned last_of_current = first_and_last_element[p][1];
1915  if (first_of_current > last_of_current)
1916  {
1917  wrong = true;
1918  error_stream << "Error: First/last element of proc " << p << ": "
1919  << first_of_current << " " << last_of_current
1920  << std::endl;
1921  }
1922  unsigned first_of_next = first_and_last_element[p + 1][0];
1923  if (first_of_next != (last_of_current + 1))
1924  {
1925  wrong = true;
1926  error_stream << "Error: First element of proc " << p + 1 << ": "
1927  << first_of_next << " and last element of proc " << p
1928  << ": " << last_of_current << std::endl;
1929  }
1930  }
1931  if (wrong)
1932  {
1933  throw OomphLibError(
1934  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
1935  }
1936 
1937 
1938  // THIS TIDY UP SHOULD NO LONGER BE REQUIRED AND CAN GO AT SOME POINT
1939 
1940  // //If we haven't got to the end of the processor list then
1941  // //need to shift things about slightly because the processors
1942  // //at the end will be empty.
1943  // //This can occur when you have very fast assembly times and the
1944  // //rounding errors mean that the targets are achieved before all
1945  // processors
1946  // //have been visited.
1947  // //Happens a lot when you massively oversubscribe the CPUs (which was
1948  // //only ever for testing!)
1949  // if (proc!=n_proc-1)
1950  // {
1951  // oomph_info
1952  // << "First pass did not allocate elements on every processor\n";
1953  // oomph_info <<
1954  // "Moving elements so that each processor has at least one\n";
1955 
1956  // //Work out number of empty processos
1957  // unsigned n_empty_processors = n_proc - proc + 1;
1958 
1959  // //Loop over the processors that do have elements
1960  // //and work out how many we need to steal elements from
1961  // unsigned n_element_on_processors=0;
1962  // do
1963  // {
1964  // //Step down the processors
1965  // --proc;
1966  // //Add the current processor to the number of empty processors
1967  // //because the elements have to be shared between processors
1968  // //including the one(s) on which they are currently stored.
1969  // ++n_empty_processors;
1970  // n_element_on_processors +=
1971  // (first_and_last_element[proc][1] -
1972  // first_and_last_element[proc][0] + 1);
1973  // }
1974  // while(n_element_on_processors < n_empty_processors);
1975 
1976  // //Should now be able to put one element on each processor
1977  // //Start from the end and do so
1978  // unsigned current_element = n_elements-1;
1979  // for(int p=n_proc-1;p>proc;p--)
1980  // {
1981  // first_and_last_element[p][1] = current_element;
1982  // first_and_last_element[p][0] = --current_element;
1983  // }
1984 
1985  // //Now for the last processor we touched, just adjust the final value
1986  // first_and_last_element[proc][1] = current_element;
1987  // }
1988  // //Otherwise just put the rest of the elements on the final
1989  // //processor
1990  // else
1991  // {
1992  // // Last one
1993  // first_and_last_element[n_proc-1][1]=n_elements-1;
1994  // }
1995 
1996 
1997  // END PRESUMED-TO-BE-UNNECESSARY BLOCK...
1998 
1999 
2000  // Now communicate the information
2001 
2002  // Set local informationt for this (root) processor
2003  First_el_for_assembly[0] = first_and_last_element[0][0];
2004  Last_el_plus_one_for_assembly[0] = first_and_last_element[0][1] + 1;
2005 
2007  {
2008  oomph_info << "Processor " << 0 << " assembles Jacobians"
2009  << " from elements " << first_and_last_element[0][0]
2010  << " to " << first_and_last_element[0][1] << " "
2011  << std::endl;
2012  }
2013 
2014  // Only now can we send the information to the other processors
2015  for (int p = 1; p < n_proc; ++p)
2016  {
2017  MPI_Send(&first_and_last_element[p][0],
2018  2,
2019  MPI_INT,
2020  p,
2021  0,
2022  this->communicator_pt()->mpi_comm());
2023 
2024 
2026  {
2027  oomph_info << "Processor " << p << " assembles Jacobians"
2028  << " from elements " << first_and_last_element[p][0]
2029  << " to " << first_and_last_element[p][1] << " "
2030  << std::endl;
2031  }
2032  }
2033  }
2034  // Receive first and last element from root on non-master processors
2035  else
2036  {
2037  Vector<int> aux(2);
2038  MPI_Status status;
2039  MPI_Recv(&aux[0],
2040  2,
2041  MPI_INT,
2042  0,
2043  0,
2044  this->communicator_pt()->mpi_comm(),
2045  &status);
2046  First_el_for_assembly[rank] = aux[0];
2047  Last_el_plus_one_for_assembly[rank] = aux[1] + 1;
2048  }
2049 
2050  // Wipe all others
2051  for (int p = 0; p < n_proc; p++)
2052  {
2053  if (p != rank)
2054  {
2055  First_el_for_assembly[p] = 0;
2057  }
2058  }
2059 
2060  // The equations assembled by this processor may have changed so
2061  // we must resize the sparse assemble with arrays previous allocation
2063  }
2064 
2065 #endif
2066 
2067  //================================================================
2068  /// Assign all equation numbers for problem: Deals with global
2069  /// data (= data that isn't attached to any elements) and then
2070  /// does the equation numbering for the elements. Bool argument
2071  /// can be set to false to ignore assigning local equation numbers
2072  /// (necessary in the parallel implementation of locate_zeta
2073  /// between multiple meshes).
2074  //================================================================
2076  const bool& assign_local_eqn_numbers)
2077  {
2078  // Check that the global mesh has been build
2079 #ifdef PARANOID
2080  if (Mesh_pt == 0)
2081  {
2082  std::ostringstream error_stream;
2083  error_stream << "Global mesh does not exist, so equation numbers cannot "
2084  "be assigned.\n";
2085  // Check for sub meshes
2086  if (nsub_mesh() == 0)
2087  {
2088  error_stream << "There aren't even any sub-meshes in the Problem.\n"
2089  << "You can set the global mesh directly by using\n"
2090  << "Problem::mesh_pt() = my_mesh_pt;\n"
2091  << "OR you can use Problem::add_sub_mesh(mesh_pt); "
2092  << "to add a sub mesh.\n";
2093  }
2094  else
2095  {
2096  error_stream << "There are " << nsub_mesh() << " sub-meshes.\n";
2097  }
2098  error_stream << "You need to call Problem::build_global_mesh() to create "
2099  "a global mesh\n"
2100  << "from the sub-meshes.\n\n";
2101 
2102  throw OomphLibError(
2103  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
2104  }
2105 #endif
2106 
2107  // Number of submeshes
2108  unsigned n_sub_mesh = Sub_mesh_pt.size();
2109 
2110 #ifdef OOMPH_HAS_MPI
2111 
2112  // Storage for number of processors
2113  int n_proc = this->communicator_pt()->nproc();
2114 
2115 
2116  if (n_proc > 1)
2117  {
2118  // Force re-analysis of time spent on assembly each
2119  // elemental Jacobian
2121  Elemental_assembly_time.clear();
2122  }
2123  else
2124  {
2126  }
2127 
2128  // Re-distribution of elements over processors during assembly
2129  // must be recomputed
2131  {
2132  // Set default first and last elements for parallel assembly
2133  // of non-distributed problem.
2135  }
2136 
2137 #endif
2138 
2139 
2140  double t_start = 0.0;
2142  {
2143  t_start = TimingHelpers::timer();
2144  }
2145 
2146  // Loop over all elements in the mesh and set up any additional
2147  // dependencies that they may have (e.g. storing the geometric
2148  // Data, i.e. Data that affects an element's shape in elements
2149  // with algebraic node-update functions
2150  unsigned nel = Mesh_pt->nelement();
2151  for (unsigned e = 0; e < nel; e++)
2152  {
2154  }
2155 
2156 #ifdef OOMPH_HAS_MPI
2157  // Complete setup of dependencies for external halo elements too
2158  unsigned n_mesh = this->nsub_mesh();
2159  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
2160  {
2161  for (int iproc = 0; iproc < n_proc; iproc++)
2162  {
2163  unsigned n_ext_halo_el = mesh_pt(i_mesh)->nexternal_halo_element(iproc);
2164  for (unsigned e = 0; e < n_ext_halo_el; e++)
2165  {
2166  mesh_pt(i_mesh)
2167  ->external_halo_element_pt(iproc, e)
2169  }
2170  }
2171  }
2172 #endif
2173 
2174 
2175  double t_end = 0.0;
2177  {
2178  t_end = TimingHelpers::timer();
2179  oomph_info
2180  << "Time for complete setup of dependencies in assign_eqn_numbers: "
2181  << t_end - t_start << std::endl;
2182  }
2183 
2184 
2185  // Initialise number of dofs for reserve below
2186  unsigned n_dof = 0;
2187 
2188  // Potentially loop over remainder of routine, possible re-visiting all
2189  // those parts that must be redone, following the removal of duplicate
2190  // external halo data.
2191  for (unsigned loop_count = 0; loop_count < 2; loop_count++)
2192  {
2193  //(Re)-set the dof pointer to zero length because entries are
2194  // pushed back onto it -- if it's not reset here then we get into
2195  // trouble during mesh refinement when we reassign all dofs
2196  Dof_pt.resize(0);
2197 
2198  // Reserve from previous allocation if we're going around again
2199  Dof_pt.reserve(n_dof);
2200 
2201  // Reset the equation number
2202  unsigned long equation_number = 0;
2203 
2204  // Now set equation numbers for the global Data
2205  unsigned Nglobal_data = nglobal_data();
2206  for (unsigned i = 0; i < Nglobal_data; i++)
2207  {
2208  Global_data_pt[i]->assign_eqn_numbers(equation_number, Dof_pt);
2209  }
2210 
2212  {
2213  t_start = TimingHelpers::timer();
2214  }
2215 
2216  // Call assign equation numbers on the global mesh
2218 
2219  // Deal with the spine meshes additional numbering
2220  // If there is only one mesh
2221  if (n_sub_mesh == 0)
2222  {
2223  if (SpineMesh* const spine_mesh_pt = dynamic_cast<SpineMesh*>(Mesh_pt))
2224  {
2225  n_dof = spine_mesh_pt->assign_global_spine_eqn_numbers(Dof_pt);
2226  }
2227  }
2228  // Otherwise loop over the sub meshes
2229  else
2230  {
2231  // Assign global equation numbers first
2232  for (unsigned i = 0; i < n_sub_mesh; i++)
2233  {
2234  if (SpineMesh* const spine_mesh_pt =
2235  dynamic_cast<SpineMesh*>(Sub_mesh_pt[i]))
2236  {
2237  n_dof = spine_mesh_pt->assign_global_spine_eqn_numbers(Dof_pt);
2238  }
2239  }
2240  }
2241 
2243  {
2244  t_end = TimingHelpers::timer();
2245  oomph_info
2246  << "Time for assign_global_eqn_numbers in assign_eqn_numbers: "
2247  << t_end - t_start << std::endl;
2248  t_start = TimingHelpers::timer();
2249  }
2250 
2251 
2252 #ifdef OOMPH_HAS_MPI
2253 
2254  // reset previous allocation
2256 
2257  // Only synchronise if the problem has actually been
2258  // distributed.
2260  {
2261  // Synchronise the equation numbers and return the total
2262  // number of degrees of freedom in the overall problem
2263  // Do not assign local equation numbers -- we're doing this
2264  // below.
2265  n_dof = synchronise_eqn_numbers(false);
2266  }
2267  // ..else just setup the Dof_distribution_pt
2268  // NOTE: this is setup by synchronise_eqn_numbers(...)
2269  // if Problem_has_been_distributed
2270  else
2271 #endif
2272  {
2273  Dof_distribution_pt->build(Communicator_pt, n_dof, false);
2274  }
2275 
2277  {
2278  t_end = TimingHelpers::timer();
2279  oomph_info << "Time for Problem::synchronise_eqn_numbers in "
2280  << "Problem::assign_eqn_numbers: " << t_end - t_start
2281  << std::endl;
2282  }
2283 
2284 
2285 #ifdef OOMPH_HAS_MPI
2286 
2287 
2288  // Now remove duplicate data in external halo elements
2290  {
2292  {
2293  t_start = TimingHelpers::timer();
2294  }
2295 
2296  // Monitor if we've actually changed anything
2297  bool actually_removed_some_data = false;
2298 
2299  // Only do it once!
2300  if (loop_count == 0)
2301  {
2302  if (n_sub_mesh == 0)
2303  {
2304  remove_duplicate_data(Mesh_pt, actually_removed_some_data);
2305  }
2306  else
2307  {
2308  for (unsigned i = 0; i < n_sub_mesh; i++)
2309  {
2310  bool tmp_actually_removed_some_data = false;
2312  tmp_actually_removed_some_data);
2313  if (tmp_actually_removed_some_data)
2314  actually_removed_some_data = true;
2315  }
2316  }
2317  }
2318 
2319 
2321  {
2322  t_end = TimingHelpers::timer();
2323  std::stringstream tmp;
2324  tmp << "Time for calls to Problem::remove_duplicate_data in "
2325  << "Problem::assign_eqn_numbers: " << t_end - t_start
2326  << " ; have ";
2327  if (!actually_removed_some_data)
2328  {
2329  tmp << " not ";
2330  }
2331  tmp << " removed some/any data.\n";
2332  oomph_info << tmp.str();
2333  t_start = TimingHelpers::timer();
2334  }
2335 
2336  // Break out of the loop if we haven't done anything here.
2337  unsigned status = 0;
2338  if (actually_removed_some_data) status = 1;
2339 
2340  // Allreduce to check if anyone has removed any data
2341  unsigned overall_status = 0;
2342  MPI_Allreduce(&status,
2343  &overall_status,
2344  1,
2345  MPI_UNSIGNED,
2346  MPI_MAX,
2347  this->communicator_pt()->mpi_comm());
2348 
2349 
2351  {
2352  t_end = TimingHelpers::timer();
2353  std::stringstream tmp;
2354  tmp
2355  << "Time for MPI_Allreduce after Problem::remove_duplicate_data in "
2356  << "Problem::assign_eqn_numbers: " << t_end - t_start << std::endl;
2357  oomph_info << tmp.str();
2358  t_start = TimingHelpers::timer();
2359  }
2360 
2361  // Bail out if we haven't done anything here
2362  if (overall_status != 1)
2363  {
2364  break;
2365  }
2366 
2367  // Big tidy up: Remove null pointers from halo/haloed node storage
2368  // for all meshes (this involves comms and therefore must be
2369  // performed outside loop over meshes so the all-to-all is only
2370  // done once)
2372 
2373  // Time it...
2375  {
2376  double t_end = TimingHelpers::timer();
2377  oomph_info << "Total time for "
2378  << "Problem::remove_null_pointers_from_external_halo_node_"
2379  "storage(): "
2380  << t_end - t_start << std::endl;
2381  }
2382  }
2383  else
2384  {
2385  // Problem not distributed; no need for another loop
2386  break;
2387  }
2388 
2389 #else
2390 
2391  // Serial run: Again no need for a second loop
2392  break;
2393 
2394 #endif
2395 
2396  } // end of loop over fcts that need to be re-executed if
2397  // we've removed duplicate external data
2398 
2399 
2400  // Resize the sparse assemble with arrays previous allocation
2402 
2403 
2405  {
2406  t_start = TimingHelpers::timer();
2407  }
2408 
2409  // Finally assign local equations
2410  if (assign_local_eqn_numbers)
2411  {
2412  if (n_sub_mesh == 0)
2413  {
2415  }
2416  else
2417  {
2418  for (unsigned i = 0; i < n_sub_mesh; i++)
2419  {
2420  Sub_mesh_pt[i]->assign_local_eqn_numbers(
2422  }
2423  }
2424  }
2425 
2427  {
2428  t_end = TimingHelpers::timer();
2429  oomph_info << "Total time for all Mesh::assign_local_eqn_numbers in "
2430  << "Problem::assign_eqn_numbers: " << t_end - t_start
2431  << std::endl;
2432  }
2433 
2434 
2435  // and return the total number of DOFs
2436  return n_dof;
2437  }
2438  //================================================================
2439  /// Function to describe the dofs in terms of the global
2440  /// equation number, i.e. what type of value (nodal value of
2441  /// a Node; value in a Data object; value of internal Data in an
2442  /// element; etc) is the unknown with a certain global equation number.
2443  /// Output stream defaults to oomph_info.
2444  //================================================================
2445  void Problem::describe_dofs(std::ostream& out) const
2446  {
2447  // Check that the global mesh has been build
2448 #ifdef PARANOID
2449  if (Mesh_pt == 0)
2450  {
2451  std::ostringstream error_stream;
2452  error_stream
2453  << "Global mesh does not exist, so equation numbers cannot be found.\n";
2454  // Check for sub meshes
2455  if (nsub_mesh() == 0)
2456  {
2457  error_stream << "There aren't even any sub-meshes in the Problem.\n"
2458  << "You can set the global mesh directly by using\n"
2459  << "Problem::mesh_pt() = my_mesh_pt;\n"
2460  << "OR you can use Problem::add_sub_mesh(mesh_pt); "
2461  << "to add a sub mesh.\n";
2462  }
2463  else
2464  {
2465  error_stream << "There are " << nsub_mesh() << " sub-meshes.\n";
2466  }
2467  error_stream << "You need to call Problem::build_global_mesh() to create "
2468  "a global mesh\n"
2469  << "from the sub-meshes.\n\n";
2470 
2471  throw OomphLibError(
2472  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
2473  }
2474 #endif
2475 
2476  out
2477  << "Although this program will describe the degrees of freedom in the \n"
2478  << "problem, it will do so using the typedef for the elements. This is \n"
2479  << "not neccesarily human readable, but there is a solution.\n"
2480  << "Pipe your program's output through c++filt, with the argument -t.\n"
2481  << "e.g. \"./two_d_multi_poisson | c++filt -t > ReadableOutput.txt\".\n "
2482  << "(Disregarding the quotes)\n\n\n";
2483 
2484  out << "Classifying Global Equation Numbers" << std::endl;
2485  out << std::string(80, '-') << std::endl;
2486 
2487  // Number of submeshes
2488  unsigned n_sub_mesh = Sub_mesh_pt.size();
2489 
2490  // Classify Global dofs
2491  unsigned Nglobal_data = nglobal_data();
2492  for (unsigned i = 0; i < Nglobal_data; i++)
2493  {
2494  std::stringstream conversion;
2495  conversion << " in Global Data " << i << ".";
2496  std::string in(conversion.str());
2497  Global_data_pt[i]->describe_dofs(out, in);
2498  }
2499 
2500  // Put string in limiting scope.
2501  {
2502  // Descend into assignment for mesh.
2503  std::string in(" in Problem's Only Mesh.");
2504  Mesh_pt->describe_dofs(out, in);
2505  }
2506 
2507  // Deal with the spine meshes additional numbering:
2508  // If there is only one mesh:
2509  if (n_sub_mesh == 0)
2510  {
2511  if (SpineMesh* const spine_mesh_pt = dynamic_cast<SpineMesh*>(Mesh_pt))
2512  {
2513  std::string in(" in Problem's Only SpineMesh.");
2514  spine_mesh_pt->describe_spine_dofs(out, in);
2515  }
2516  }
2517  // Otherwise loop over the sub meshes
2518  else
2519  {
2520  // Assign global equation numbers first
2521  for (unsigned i = 0; i < n_sub_mesh; i++)
2522  {
2523  if (SpineMesh* const spine_mesh_pt =
2524  dynamic_cast<SpineMesh*>(Sub_mesh_pt[i]))
2525  {
2526  std::stringstream conversion;
2527  conversion << " in Sub-SpineMesh " << i << ".";
2528  std::string in(conversion.str());
2529  spine_mesh_pt->describe_spine_dofs(out, in);
2530  } // end if.
2531  } // end for.
2532  } // end else.
2533 
2534 
2535  out << std::string(80, '\\') << std::endl;
2536  out << std::string(80, '\\') << std::endl;
2537  out << std::string(80, '\\') << std::endl;
2538  out << "Classifying global eqn numbers in terms of elements." << std::endl;
2539  out << std::string(80, '-') << std::endl;
2540  out << "Eqns | Source" << std::endl;
2541  out << std::string(80, '-') << std::endl;
2542 
2543  if (n_sub_mesh == 0)
2544  {
2545  std::string in(" in Problem's Only Mesh.");
2546  Mesh_pt->describe_local_dofs(out, in);
2547  }
2548  else
2549  {
2550  for (unsigned i = 0; i < n_sub_mesh; i++)
2551  {
2552  std::stringstream conversion;
2553  conversion << " in Sub-Mesh " << i << ".";
2554  std::string in(conversion.str());
2555  Sub_mesh_pt[i]->describe_local_dofs(out, in);
2556  } // End for
2557  } // End else
2558  } // End problem::describe_dofs(...)
2559 
2560 
2561  //================================================================
2562  /// Get the vector of dofs, i.e. a vector containing the current
2563  /// values of all unknowns.
2564  //================================================================
2566  {
2567  // Find number of dofs
2568  const unsigned long n_dof = ndof();
2569 
2570  // Resize the vector
2571  dofs.build(Dof_distribution_pt, 0.0);
2572 
2573  // Copy dofs into vector
2574  for (unsigned long l = 0; l < n_dof; l++)
2575  {
2576  dofs[l] = *Dof_pt[l];
2577  }
2578  }
2579 
2580  /// Get history values of dofs in a double vector.
2581  void Problem::get_dofs(const unsigned& t, DoubleVector& dofs) const
2582  {
2583 #ifdef PARANOID
2584  if (distributed())
2585  {
2586  throw OomphLibError("Not designed for distributed problems",
2587  OOMPH_EXCEPTION_LOCATION,
2588  OOMPH_CURRENT_FUNCTION);
2589  // might work, not sure
2590  }
2591 #endif
2592 
2593  // Resize the vector
2594  dofs.build(Dof_distribution_pt, 0.0);
2595 
2596  // First deal with global data
2597  unsigned Nglobal_data = nglobal_data();
2598  for (unsigned i = 0; i < Nglobal_data; i++)
2599  {
2600  for (unsigned j = 0, nj = Global_data_pt[i]->nvalue(); j < nj; j++)
2601  {
2602  // For each data get the equation number and copy out the value.
2603  int eqn_number = Global_data_pt[i]->eqn_number(j);
2604  if (eqn_number >= 0)
2605  {
2606  dofs[eqn_number] = Global_data_pt[i]->value(t, j);
2607  }
2608  }
2609  }
2610 
2611  // Next element internal data
2612  for (unsigned i = 0, ni = mesh_pt()->nelement(); i < ni; i++)
2613  {
2614  GeneralisedElement* ele_pt = mesh_pt()->element_pt(i);
2615  for (unsigned j = 0, nj = ele_pt->ninternal_data(); j < nj; j++)
2616  {
2617  Data* d_pt = ele_pt->internal_data_pt(j);
2618  for (unsigned k = 0, nk = d_pt->nvalue(); k < nk; k++)
2619  {
2620  int eqn_number = d_pt->eqn_number(k);
2621  if (eqn_number >= 0)
2622  {
2623  dofs[eqn_number] = d_pt->value(t, k);
2624  }
2625  }
2626  }
2627  }
2628 
2629  // Now the nodes
2630  for (unsigned i = 0, ni = mesh_pt()->nnode(); i < ni; i++)
2631  {
2632  Node* node_pt = mesh_pt()->node_pt(i);
2633  for (unsigned j = 0, nj = node_pt->nvalue(); j < nj; j++)
2634  {
2635  // For each node get the equation number and copy out the value.
2636  int eqn_number = node_pt->eqn_number(j);
2637  if (eqn_number >= 0)
2638  {
2639  dofs[eqn_number] = node_pt->value(t, j);
2640  }
2641  }
2642  }
2643  }
2644 
2645 
2646 #ifdef OOMPH_HAS_MPI
2647 
2648  //=======================================================================
2649  /// Private helper function to remove repeated data
2650  /// in external haloed elements associated with specified mesh.
2651  /// Bool is true if some data was removed -- this usually requires
2652  /// re-running through certain parts of the equation numbering procedure.
2653  //======================================================================
2654  void Problem::remove_duplicate_data(Mesh* const& mesh_pt,
2655  bool& actually_removed_some_data)
2656  {
2657  // // // Taken out again by MH -- clutters up output
2658  // // Doc timings if required
2659  // double t_start=0.0;
2660  // if (Global_timings::Doc_comprehensive_timings)
2661  // {
2662  // t_start=TimingHelpers::timer();
2663  // }
2664 
2665  int n_proc = this->communicator_pt()->nproc();
2666  int my_rank = this->communicator_pt()->my_rank();
2667 
2668  // Initialise
2669  actually_removed_some_data = false;
2670 
2671  // Each individual container of external halo nodes has unique
2672  // nodes/equation numbers, but there may be some duplication between
2673  // two or more different containers; the following code checks for this
2674  // and removes the duplication by overwriting any data point with an already
2675  // existing eqn number with the original data point which had the eqn no.
2676 
2677  // // Storage for existing nodes, enumerated by first non-negative
2678  // // global equation number
2679  // unsigned n_dof=ndof();
2680 
2681  // Note: This used to be
2682  // Vector<Node*> global_node_pt(n_dof,0);
2683  // but this is a total killer! Memory allocation is extremely
2684  // costly and only relatively few entries are used so use
2685  // map:
2686  std::map<unsigned, Node*> global_node_pt;
2687 
2688  // Only do each retained node once
2689  std::map<Node*, bool> node_done;
2690 
2691  // Loop over existing "normal" elements in mesh
2692  unsigned n_element = mesh_pt->nelement();
2693  for (unsigned e = 0; e < n_element; e++)
2694  {
2695  FiniteElement* el_pt =
2696  dynamic_cast<FiniteElement*>(mesh_pt->element_pt(e));
2697  if (el_pt != 0)
2698  {
2699  // Loop over nodes
2700  unsigned n_node = el_pt->nnode();
2701  for (unsigned j = 0; j < n_node; j++)
2702  {
2703  Node* nod_pt = el_pt->node_pt(j);
2704 
2705  // Have we already done the node?
2706  if (!node_done[nod_pt])
2707  {
2708  node_done[nod_pt] = true;
2709 
2710  // Loop over values stored at node (if any) to find
2711  // the first non-negative eqn number
2712  unsigned first_non_negative_eqn_number_plus_one = 0;
2713  unsigned n_val = nod_pt->nvalue();
2714  for (unsigned i_val = 0; i_val < n_val; i_val++)
2715  {
2716  int eqn_no = nod_pt->eqn_number(i_val);
2717  if (eqn_no >= 0)
2718  {
2719  first_non_negative_eqn_number_plus_one = eqn_no + 1;
2720  break;
2721  }
2722  }
2723 
2724  // If we haven't found a non-negative eqn number check
2725  // eqn numbers associated with solid data (if any)
2726  if (first_non_negative_eqn_number_plus_one == 0)
2727  {
2728  // Is it a solid node?
2729  SolidNode* solid_nod_pt = dynamic_cast<SolidNode*>(nod_pt);
2730  if (solid_nod_pt != 0)
2731  {
2732  // Loop over values stored at node (if any) to find
2733  // the first non-negative eqn number
2734  unsigned n_val = solid_nod_pt->variable_position_pt()->nvalue();
2735  for (unsigned i_val = 0; i_val < n_val; i_val++)
2736  {
2737  int eqn_no =
2738  solid_nod_pt->variable_position_pt()->eqn_number(i_val);
2739  if (eqn_no >= 0)
2740  {
2741  first_non_negative_eqn_number_plus_one = eqn_no + 1;
2742  break;
2743  }
2744  }
2745  }
2746  }
2747 
2748  // Associate node with first non negative global eqn number
2749  if (first_non_negative_eqn_number_plus_one > 0)
2750  {
2751  global_node_pt[first_non_negative_eqn_number_plus_one - 1] =
2752  nod_pt;
2753  }
2754 
2755 
2756  // Take into account master nodes too
2757  if (dynamic_cast<RefineableElement*>(el_pt) != 0)
2758  {
2759  int n_cont_int_values = dynamic_cast<RefineableElement*>(el_pt)
2760  ->ncont_interpolated_values();
2761  for (int i_cont = -1; i_cont < n_cont_int_values; i_cont++)
2762  {
2763  if (nod_pt->is_hanging(i_cont))
2764  {
2765  HangInfo* hang_pt = nod_pt->hanging_pt(i_cont);
2766  unsigned n_master = hang_pt->nmaster();
2767  for (unsigned m = 0; m < n_master; m++)
2768  {
2769  Node* master_nod_pt = hang_pt->master_node_pt(m);
2770  if (!node_done[master_nod_pt])
2771  {
2772  node_done[master_nod_pt] = true;
2773 
2774  // Loop over values stored at node (if any) to find
2775  // the first non-negative eqn number
2776  unsigned first_non_negative_eqn_number_plus_one = 0;
2777  unsigned n_val = master_nod_pt->nvalue();
2778  for (unsigned i_val = 0; i_val < n_val; i_val++)
2779  {
2780  int eqn_no = master_nod_pt->eqn_number(i_val);
2781  if (eqn_no >= 0)
2782  {
2783  first_non_negative_eqn_number_plus_one = eqn_no + 1;
2784  break;
2785  }
2786  }
2787 
2788  // If we haven't found a non-negative eqn number check
2789  // eqn numbers associated with solid data (if any)
2790  if (first_non_negative_eqn_number_plus_one == 0)
2791  {
2792  // If this master is a SolidNode then add its extra
2793  // eqn numbers
2794  SolidNode* master_solid_nod_pt =
2795  dynamic_cast<SolidNode*>(master_nod_pt);
2796  if (master_solid_nod_pt != 0)
2797  {
2798  // Loop over values stored at node (if any) to find
2799  // the first non-negative eqn number
2800  unsigned n_val =
2801  master_solid_nod_pt->variable_position_pt()
2802  ->nvalue();
2803  for (unsigned i_val = 0; i_val < n_val; i_val++)
2804  {
2805  int eqn_no =
2806  master_solid_nod_pt->variable_position_pt()
2807  ->eqn_number(i_val);
2808  if (eqn_no >= 0)
2809  {
2810  first_non_negative_eqn_number_plus_one =
2811  eqn_no + 1;
2812  break;
2813  }
2814  }
2815  }
2816  }
2817  // Associate node with first non negative global
2818  // eqn number
2819  if (first_non_negative_eqn_number_plus_one > 0)
2820  {
2821  global_node_pt[first_non_negative_eqn_number_plus_one -
2822  1] = master_nod_pt;
2823  }
2824 
2825  } // End of not-yet-done hang node
2826  }
2827  }
2828  }
2829  }
2830  } // endif for node already done
2831  } // End of loop over nodes
2832  } // End of FiniteElement
2833 
2834  // Internal data equation numbers do not need to be added since
2835  // internal data cannot be shared between distinct elements, so
2836  // internal data on locally-stored elements can never be halo.
2837  }
2838 
2839  // Set to record duplicate nodes scheduled to be killed
2840  std::set<Node*> killed_nodes;
2841 
2842  // Now loop over the other processors from highest to lowest
2843  // (i.e. if there is a duplicate between these containers
2844  // then this will use the node on the highest numbered processor)
2845  for (int iproc = n_proc - 1; iproc >= 0; iproc--)
2846  {
2847  // Don't have external halo elements with yourself!
2848  if (iproc != my_rank)
2849  {
2850  // Loop over external halo elements with iproc
2851  // to remove the duplicates
2852  unsigned n_element = mesh_pt->nexternal_halo_element(iproc);
2853  for (unsigned e_ext = 0; e_ext < n_element; e_ext++)
2854  {
2855  FiniteElement* finite_ext_el_pt = dynamic_cast<FiniteElement*>(
2856  mesh_pt->external_halo_element_pt(iproc, e_ext));
2857  if (finite_ext_el_pt != 0)
2858  {
2859  // Loop over nodes
2860  unsigned n_node = finite_ext_el_pt->nnode();
2861  for (unsigned j = 0; j < n_node; j++)
2862  {
2863  Node* nod_pt = finite_ext_el_pt->node_pt(j);
2864 
2865  // Loop over values stored at node (if any) to find
2866  // the first non-negative eqn number
2867  unsigned first_non_negative_eqn_number_plus_one = 0;
2868  unsigned n_val = nod_pt->nvalue();
2869  for (unsigned i_val = 0; i_val < n_val; i_val++)
2870  {
2871  int eqn_no = nod_pt->eqn_number(i_val);
2872  if (eqn_no >= 0)
2873  {
2874  first_non_negative_eqn_number_plus_one = eqn_no + 1;
2875  break;
2876  }
2877  }
2878 
2879  // If we haven't found a non-negative eqn number check
2880  // eqn numbers associated with solid data (if any)
2881  if (first_non_negative_eqn_number_plus_one == 0)
2882  {
2883  // Is it a solid node?
2884  SolidNode* solid_nod_pt = dynamic_cast<SolidNode*>(nod_pt);
2885  if (solid_nod_pt != 0)
2886  {
2887  // Loop over values stored at node (if any) to find
2888  // the first non-negative eqn number
2889  unsigned n_val =
2890  solid_nod_pt->variable_position_pt()->nvalue();
2891  for (unsigned i_val = 0; i_val < n_val; i_val++)
2892  {
2893  int eqn_no =
2894  solid_nod_pt->variable_position_pt()->eqn_number(i_val);
2895  if (eqn_no >= 0)
2896  {
2897  first_non_negative_eqn_number_plus_one = eqn_no + 1;
2898  break;
2899  }
2900  }
2901  }
2902  }
2903 
2904  // Identified which node we're dealing with via first non-negative
2905  // global eqn number (if there is none, everything is pinned
2906  // and we don't give a damn...)
2907  if (first_non_negative_eqn_number_plus_one > 0)
2908  {
2909  Node* existing_node_pt =
2910  global_node_pt[first_non_negative_eqn_number_plus_one - 1];
2911 
2912  // Does this node already exist?
2913  if (existing_node_pt != 0)
2914  {
2915  // Record that we're about to cull one
2916  actually_removed_some_data = true;
2917 
2918  // It's a duplicate, so store the duplicated one for
2919  // later killing...
2920  Node* duplicated_node_pt = nod_pt;
2921  if (!node_done[duplicated_node_pt])
2922  {
2923  // Remove node from all boundaries
2924  std::set<unsigned>* boundaries_pt;
2925  duplicated_node_pt->get_boundaries_pt(boundaries_pt);
2926  if (boundaries_pt != 0)
2927  {
2928  Vector<unsigned> bound;
2929  unsigned nb = (*boundaries_pt).size();
2930  bound.reserve(nb);
2931  for (std::set<unsigned>::iterator it =
2932  (*boundaries_pt).begin();
2933  it != (*boundaries_pt).end();
2934  it++)
2935  {
2936  bound.push_back((*it));
2937  }
2938  for (unsigned i = 0; i < nb; i++)
2939  {
2940  mesh_pt->remove_boundary_node(bound[i],
2941  duplicated_node_pt);
2942  }
2943  }
2944 
2945  // Get ready to kill it
2946  killed_nodes.insert(duplicated_node_pt);
2947  unsigned i_proc = unsigned(iproc);
2949  duplicated_node_pt);
2950  }
2951 
2952 
2953  // Note: For now we're leaving the "dangling" (no longer
2954  // accessed masters where they are; they get cleaned
2955  // up next time we delete all the external storage
2956  // for the meshes so it's a temporary "leak" only...
2957  // At some point we should probably delete them properly too
2958 
2959 #ifdef PARANOID
2960 
2961  // Check that hang status of exiting and replacement node
2962  // matches
2963  if (dynamic_cast<RefineableElement*>(finite_ext_el_pt) != 0)
2964  {
2965  int n_cont_inter_values =
2966  dynamic_cast<RefineableElement*>(finite_ext_el_pt)
2967  ->ncont_interpolated_values();
2968  for (int i_cont = -1; i_cont < n_cont_inter_values;
2969  i_cont++)
2970  {
2971  unsigned n_master_orig = 0;
2972  if (finite_ext_el_pt->node_pt(j)->is_hanging(i_cont))
2973  {
2974  n_master_orig = finite_ext_el_pt->node_pt(j)
2975  ->hanging_pt(i_cont)
2976  ->nmaster();
2977 
2978  // Temporary leak: Resolve like this:
2979  // loop over all external halo nodes and identify the
2980  // the ones that are still reached by any of the
2981  // external elements. Kill the dangling ones.
2982  }
2983  unsigned n_master_replace = 0;
2984  if (existing_node_pt->is_hanging(i_cont))
2985  {
2986  n_master_replace =
2987  existing_node_pt->hanging_pt(i_cont)->nmaster();
2988  }
2989 
2990  if (n_master_orig != n_master_replace)
2991  {
2992  std::ostringstream error_stream;
2993  error_stream
2994  << "Number of master nodes for node to be replaced, "
2995  << n_master_orig << ", doesn't match"
2996  << "those of replacement node, " << n_master_replace
2997  << " for i_cont=" << i_cont << std::endl;
2998  {
2999  error_stream
3000  << "Nodal coordinates of replacement node:";
3001  unsigned ndim = existing_node_pt->ndim();
3002  for (unsigned i = 0; i < ndim; i++)
3003  {
3004  error_stream << existing_node_pt->x(i) << " ";
3005  }
3006  error_stream << "\n";
3007  error_stream << "The coordinates of its "
3008  << n_master_replace
3009  << " master nodes are: \n";
3010  for (unsigned k = 0; k < n_master_replace; k++)
3011  {
3012  Node* master_nod_pt =
3013  existing_node_pt->hanging_pt(i_cont)
3014  ->master_node_pt(k);
3015  unsigned ndim = master_nod_pt->ndim();
3016  for (unsigned i = 0; i < ndim; i++)
3017  {
3018  error_stream << master_nod_pt->x(i) << " ";
3019  }
3020  error_stream << "\n";
3021  }
3022  }
3023 
3024  {
3025  error_stream
3026  << "Nodal coordinates of node to be replaced:";
3027  unsigned ndim = finite_ext_el_pt->node_pt(j)->ndim();
3028  for (unsigned i = 0; i < ndim; i++)
3029  {
3030  error_stream << finite_ext_el_pt->node_pt(j)->x(i)
3031  << " ";
3032  }
3033  error_stream << "\n";
3034  error_stream << "The coordinates of its "
3035  << n_master_orig
3036  << " master nodes are: \n";
3037  for (unsigned k = 0; k < n_master_orig; k++)
3038  {
3039  Node* master_nod_pt = finite_ext_el_pt->node_pt(j)
3040  ->hanging_pt(i_cont)
3041  ->master_node_pt(k);
3042  unsigned ndim = master_nod_pt->ndim();
3043  for (unsigned i = 0; i < ndim; i++)
3044  {
3045  error_stream << master_nod_pt->x(i) << " ";
3046  }
3047  error_stream << "\n";
3048  }
3049  }
3050 
3051 
3052  throw OomphLibError(error_stream.str(),
3053  OOMPH_CURRENT_FUNCTION,
3054  OOMPH_EXCEPTION_LOCATION);
3055  }
3056  }
3057  }
3058 #endif
3059  // ...and point to the existing one
3060  finite_ext_el_pt->node_pt(j) = existing_node_pt;
3061  }
3062  // If it doesn't add it to the list of existing ones
3063  else
3064  {
3065  global_node_pt[first_non_negative_eqn_number_plus_one - 1] =
3066  nod_pt;
3067  node_done[nod_pt] = true;
3068  }
3069  }
3070 
3071 
3072  // Do the same for any master nodes of that (possibly replaced)
3073  // node
3074  if (dynamic_cast<RefineableElement*>(finite_ext_el_pt) != 0)
3075  {
3076  int n_cont_inter_values =
3077  dynamic_cast<RefineableElement*>(finite_ext_el_pt)
3078  ->ncont_interpolated_values();
3079  for (int i_cont = -1; i_cont < n_cont_inter_values; i_cont++)
3080  {
3081  if (finite_ext_el_pt->node_pt(j)->is_hanging(i_cont))
3082  {
3083  HangInfo* hang_pt =
3084  finite_ext_el_pt->node_pt(j)->hanging_pt(i_cont);
3085  unsigned n_master = hang_pt->nmaster();
3086  for (unsigned m = 0; m < n_master; m++)
3087  {
3088  Node* master_nod_pt = hang_pt->master_node_pt(m);
3089  unsigned n_val = master_nod_pt->nvalue();
3090  unsigned first_non_negative_eqn_number_plus_one = 0;
3091  for (unsigned i_val = 0; i_val < n_val; i_val++)
3092  {
3093  int eqn_no = master_nod_pt->eqn_number(i_val);
3094  if (eqn_no >= 0)
3095  {
3096  first_non_negative_eqn_number_plus_one = eqn_no + 1;
3097  break;
3098  }
3099  }
3100 
3101  // If we haven't found a non-negative eqn number check
3102  // eqn numbers associated with solid data (if any)
3103  if (first_non_negative_eqn_number_plus_one == 0)
3104  {
3105  SolidNode* solid_master_nod_pt =
3106  dynamic_cast<SolidNode*>(master_nod_pt);
3107  if (solid_master_nod_pt != 0)
3108  {
3109  // Loop over values stored at node (if any) to find
3110  // the first non-negative eqn number
3111  unsigned n_val =
3112  solid_master_nod_pt->variable_position_pt()
3113  ->nvalue();
3114  for (unsigned i_val = 0; i_val < n_val; i_val++)
3115  {
3116  int eqn_no =
3117  solid_master_nod_pt->variable_position_pt()
3118  ->eqn_number(i_val);
3119  if (eqn_no >= 0)
3120  {
3121  first_non_negative_eqn_number_plus_one =
3122  eqn_no + 1;
3123  break;
3124  }
3125  }
3126  }
3127  }
3128 
3129  // Identified which node we're dealing with via
3130  // first non-negative global eqn number (if there
3131  // is none, everything is pinned and we don't give a
3132  // damn...)
3133  if (first_non_negative_eqn_number_plus_one > 0)
3134  {
3135  Node* existing_node_pt = global_node_pt
3136  [first_non_negative_eqn_number_plus_one - 1];
3137 
3138  // Does this node already exist?
3139  if (existing_node_pt != 0)
3140  {
3141  // Record that we're about to cull one
3142  actually_removed_some_data = true;
3143 
3144  // It's a duplicate, so store the duplicated one for
3145  // later killing...
3146  Node* duplicated_node_pt = master_nod_pt;
3147 
3148  if (!node_done[duplicated_node_pt])
3149  {
3150  // Remove node from all boundaries
3151  std::set<unsigned>* boundaries_pt;
3152  duplicated_node_pt->get_boundaries_pt(
3153  boundaries_pt);
3154  if (boundaries_pt != 0)
3155  {
3156  for (std::set<unsigned>::iterator it =
3157  (*boundaries_pt).begin();
3158  it != (*boundaries_pt).end();
3159  it++)
3160  {
3162  (*it), duplicated_node_pt);
3163  }
3164  }
3165 
3166  killed_nodes.insert(duplicated_node_pt);
3167  unsigned i_proc = unsigned(iproc);
3169  i_proc, duplicated_node_pt);
3170  }
3171 
3172  // Weight of the original node
3173  double m_weight = hang_pt->master_weight(m);
3174 
3175 
3176 #ifdef PARANOID
3177  // Sanity check: setting replacement master
3178  // node for non-hanging node? Sign of really
3179  // f***ed up code.
3180  Node* tmp_nod_pt = finite_ext_el_pt->node_pt(j);
3181  if (!tmp_nod_pt->is_hanging(i_cont))
3182  {
3183  std::ostringstream error_stream;
3184  error_stream
3185  << "About to re-set master for i_cont= " << i_cont
3186  << " for external node (with proc " << iproc
3187  << " )" << tmp_nod_pt << " at ";
3188  unsigned n = tmp_nod_pt->ndim();
3189  for (unsigned jj = 0; jj < n; jj++)
3190  {
3191  error_stream << tmp_nod_pt->x(jj) << " ";
3192  }
3193  error_stream
3194  << " which is not hanging --> About to die!"
3195  << "Outputting offending element into oomph-info "
3196  << "stream. \n\n";
3197  oomph_info << "\n\n";
3198  finite_ext_el_pt->output(*(oomph_info.stream_pt()));
3199  oomph_info << "\n\n";
3200  oomph_info.stream_pt()->flush();
3201  throw OomphLibError(error_stream.str(),
3202  OOMPH_CURRENT_FUNCTION,
3203  OOMPH_EXCEPTION_LOCATION);
3204  }
3205 #endif
3206 
3207 
3208  // And re-set master
3209  finite_ext_el_pt->node_pt(j)
3210  ->hanging_pt(i_cont)
3211  ->set_master_node_pt(m, existing_node_pt, m_weight);
3212  }
3213  // If it doesn't, add it to the list of existing ones
3214  else
3215  {
3216  global_node_pt
3217  [first_non_negative_eqn_number_plus_one - 1] =
3218  master_nod_pt;
3219  node_done[master_nod_pt] = true;
3220  }
3221  }
3222  } // End of loop over master nodes
3223  } // end of hanging
3224  } // end of loop over continously interpolated variables
3225  } // end refineable element (with potentially hanging node
3226 
3227  } // end loop over nodes on external halo elements
3228 
3229  } // End of check for finite element
3230 
3231  } // end loop over external halo elements
3232  }
3233  } // end loop over processors
3234 
3235 
3236  // Now kill all the deleted nodes
3237  for (std::set<Node*>::iterator it = killed_nodes.begin();
3238  it != killed_nodes.end();
3239  it++)
3240  {
3241  delete (*it);
3242  }
3243 
3244 
3245  // oomph_info << "Number of nonzero entries in global_node_pt: "
3246  // << global_node_pt.size() << std::endl;
3247 
3248  // // Time it...
3249  // // Taken out again by MH -- clutters up output
3250  // if (Global_timings::Doc_comprehensive_timings)
3251  // {
3252  // double t_end = TimingHelpers::timer();
3253  // oomph_info
3254  // << "Total time for Problem::remove_duplicate_data: "
3255  // << t_end-t_start << std::endl;
3256  // }
3257  }
3258 
3259 
3260  //========================================================================
3261  /// Consolidate external halo node storage by removing nulled out
3262  /// pointers in external halo and haloed schemes for all meshes.
3263  //========================================================================
3265  {
3266  // Do we have submeshes?
3267  unsigned n_mesh_loop = 1;
3268  unsigned nmesh = nsub_mesh();
3269  if (nmesh > 0)
3270  {
3271  n_mesh_loop = nmesh;
3272  }
3273 
3274  // Storage for number of processors and current processor
3275  int n_proc = this->communicator_pt()->nproc();
3276  int my_rank = this->communicator_pt()->my_rank();
3277 
3278  // If only one processor then return
3279  if (n_proc == 1)
3280  {
3281  return;
3282  }
3283 
3284  // Loop over all (other) processors and store index of any nulled-out
3285  // external halo nodes in storage scheme.
3286 
3287  // Data to be sent to each processor
3288  Vector<int> send_n(n_proc, 0);
3289 
3290  // Storage for all values to be sent to all processors
3291  Vector<int> send_data;
3292 
3293  // Start location within send_data for data to be sent to each processor
3294  Vector<int> send_displacement(n_proc, 0);
3295 
3296  // Check missing ones
3297  for (int domain = 0; domain < n_proc; domain++)
3298  {
3299  // Set the offset for the current processor
3300  send_displacement[domain] = send_data.size();
3301 
3302  // Don't bother to do anything if the processor in the loop is the
3303  // current processor
3304  if (domain != my_rank)
3305  {
3306  // Deal with sub-meshes one-by-one if required
3307  Mesh* my_mesh_pt = 0;
3308 
3309  // Loop over submeshes
3310  for (unsigned imesh = 0; imesh < n_mesh_loop; imesh++)
3311  {
3312  if (nmesh == 0)
3313  {
3314  my_mesh_pt = mesh_pt();
3315  }
3316  else
3317  {
3318  my_mesh_pt = mesh_pt(imesh);
3319  }
3320 
3321  // Make backup of external halo node pointers with this domain
3322  Vector<Node*> backup_pt(my_mesh_pt->external_halo_node_pt(domain));
3323 
3324  // How many do we have currently?
3325  unsigned nnod = backup_pt.size();
3326 
3327  // Prepare storage for updated halo nodes
3328  Vector<Node*> new_external_halo_node_pt;
3329  new_external_halo_node_pt.reserve(nnod);
3330 
3331  // Loop over external halo nodes with this domain
3332  for (unsigned j = 0; j < nnod; j++)
3333  {
3334  // Get pointer to node
3335  Node* nod_pt = backup_pt[j];
3336 
3337  // Has it been nulled out?
3338  if (nod_pt == 0)
3339  {
3340  // Save index of nulled out one
3341  send_data.push_back(j);
3342  }
3343  else
3344  {
3345  // Still alive: Copy across
3346  new_external_halo_node_pt.push_back(nod_pt);
3347  }
3348  }
3349 
3350  // Set new external halo node vector
3351  my_mesh_pt->set_external_halo_node_pt(domain,
3352  new_external_halo_node_pt);
3353 
3354  // End of data for this mesh
3355  send_data.push_back(-1);
3356 
3357  } // end of loop over meshes
3358 
3359  } // end skip own domain
3360 
3361  // Find the number of data added to the vector
3362  send_n[domain] = send_data.size() - send_displacement[domain];
3363  }
3364 
3365  // Storage for the number of data to be received from each processor
3366  Vector<int> receive_n(n_proc, 0);
3367 
3368  // Now send numbers of data to be sent between all processors
3369  MPI_Alltoall(&send_n[0],
3370  1,
3371  MPI_INT,
3372  &receive_n[0],
3373  1,
3374  MPI_INT,
3375  this->communicator_pt()->mpi_comm());
3376 
3377 
3378  // We now prepare the data to be received
3379  // by working out the displacements from the received data
3380  Vector<int> receive_displacement(n_proc, 0);
3381  int receive_data_count = 0;
3382  for (int rank = 0; rank < n_proc; ++rank)
3383  {
3384  // Displacement is number of data received so far
3385  receive_displacement[rank] = receive_data_count;
3386  receive_data_count += receive_n[rank];
3387  }
3388 
3389  // Now resize the receive buffer for all data from all processors
3390  // Make sure that it has a size of at least one
3391  if (receive_data_count == 0)
3392  {
3393  ++receive_data_count;
3394  }
3395  Vector<int> receive_data(receive_data_count);
3396 
3397  // Make sure that the send buffer has size at least one
3398  // so that we don't get a segmentation fault
3399  if (send_data.size() == 0)
3400  {
3401  send_data.resize(1);
3402  }
3403 
3404  // Now send the data between all the processors
3405  MPI_Alltoallv(&send_data[0],
3406  &send_n[0],
3407  &send_displacement[0],
3408  MPI_INT,
3409  &receive_data[0],
3410  &receive_n[0],
3411  &receive_displacement[0],
3412  MPI_INT,
3413  this->communicator_pt()->mpi_comm());
3414 
3415  // Now use the received data
3416  for (int send_rank = 0; send_rank < n_proc; send_rank++)
3417  {
3418  // Don't bother to do anything for the processor corresponding to the
3419  // current processor or if no data were received from this processor
3420  if ((send_rank != my_rank) && (receive_n[send_rank] != 0))
3421  {
3422  // Counter for the data within the large array
3423  unsigned count = receive_displacement[send_rank];
3424 
3425  // Deal with sub-meshes one-by-one if required
3426  Mesh* my_mesh_pt = 0;
3427 
3428  // Loop over submeshes
3429  for (unsigned imesh = 0; imesh < n_mesh_loop; imesh++)
3430  {
3431  if (nmesh == 0)
3432  {
3433  my_mesh_pt = mesh_pt();
3434  }
3435  else
3436  {
3437  my_mesh_pt = mesh_pt(imesh);
3438  }
3439 
3440  // Make backup of external haloed node pointers with this domain
3441  Vector<Node*> backup_pt =
3442  my_mesh_pt->external_haloed_node_pt(send_rank);
3443 
3444  // Unpack until we reach "end of data" indicator (-1) for this mesh
3445  while (true)
3446  {
3447  // Read next entry
3448  int next_one = receive_data[count++];
3449 
3450  if (next_one == -1)
3451  {
3452  break;
3453  }
3454  else
3455  {
3456  // Null out the entry
3457  backup_pt[next_one] = 0;
3458  }
3459  }
3460 
3461  // How many do we have currently?
3462  unsigned nnod = backup_pt.size();
3463 
3464  // Prepare storage for updated haloed nodes
3465  Vector<Node*> new_external_haloed_node_pt;
3466  new_external_haloed_node_pt.reserve(nnod);
3467 
3468  // Loop over external haloed nodes with this domain
3469  for (unsigned j = 0; j < nnod; j++)
3470  {
3471  // Get pointer to node
3472  Node* nod_pt = backup_pt[j];
3473 
3474  // Has it been nulled out?
3475  if (nod_pt != 0)
3476  {
3477  // Still alive: Copy across
3478  new_external_haloed_node_pt.push_back(nod_pt);
3479  }
3480  }
3481 
3482  // Set new external haloed node vector
3483  my_mesh_pt->set_external_haloed_node_pt(send_rank,
3484  new_external_haloed_node_pt);
3485  }
3486  }
3487 
3488  } // End of data is received
3489  }
3490 
3491 #endif
3492 
3493 
3494  //=======================================================================
3495  /// Function that sets the values of the dofs in the object
3496  //======================================================================
3498  {
3499  const unsigned long n_dof = this->ndof();
3500 #ifdef PARANOID
3501  if (n_dof != dofs.nrow())
3502  {
3503  std::ostringstream error_stream;
3504  error_stream << "Number of degrees of freedom in vector argument "
3505  << dofs.nrow() << "\n"
3506  << "does not equal number of degrees of freedom in problem "
3507  << n_dof;
3508  throw OomphLibError(
3509  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
3510  }
3511 #endif
3512  for (unsigned long l = 0; l < n_dof; l++)
3513  {
3514  *Dof_pt[l] = dofs[l];
3515  }
3516  }
3517 
3518  /// Set history values of dofs
3519  void Problem::set_dofs(const unsigned& t, DoubleVector& dofs)
3520  {
3521 #ifdef PARANOID
3522  if (distributed())
3523  {
3524  throw OomphLibError("Not designed for distributed problems",
3525  OOMPH_EXCEPTION_LOCATION,
3526  OOMPH_CURRENT_FUNCTION);
3527  // might work if the dofs vector is distributed in the right way...
3528  }
3529 #endif
3530 
3531  // First deal with global data
3532  unsigned Nglobal_data = nglobal_data();
3533  for (unsigned i = 0; i < Nglobal_data; i++)
3534  {
3535  for (unsigned j = 0, nj = Global_data_pt[i]->nvalue(); j < nj; j++)
3536  {
3537  // For each data get the equation number and copy out the value.
3538  int eqn_number = Global_data_pt[i]->eqn_number(j);
3539  if (eqn_number >= 0)
3540  {
3541  Global_data_pt[i]->set_value(t, j, dofs[eqn_number]);
3542  }
3543  }
3544  }
3545 
3546  // Next element internal data
3547  for (unsigned i = 0, ni = mesh_pt()->nelement(); i < ni; i++)
3548  {
3549  GeneralisedElement* ele_pt = mesh_pt()->element_pt(i);
3550  for (unsigned j = 0, nj = ele_pt->ninternal_data(); j < nj; j++)
3551  {
3552  Data* d_pt = ele_pt->internal_data_pt(j);
3553  for (unsigned k = 0, nk = d_pt->nvalue(); k < nk; k++)
3554  {
3555  int eqn_number = d_pt->eqn_number(k);
3556  if (eqn_number >= 0)
3557  {
3558  d_pt->set_value(t, k, dofs[eqn_number]);
3559  }
3560  }
3561  }
3562  }
3563 
3564  // Now the nodes
3565  for (unsigned i = 0, ni = mesh_pt()->nnode(); i < ni; i++)
3566  {
3567  Node* node_pt = mesh_pt()->node_pt(i);
3568  for (unsigned j = 0, nj = node_pt->nvalue(); j < nj; j++)
3569  {
3570  // For each node get the equation number and copy out the value.
3571  int eqn_number = node_pt->eqn_number(j);
3572  if (eqn_number >= 0)
3573  {
3574  node_pt->set_value(t, j, dofs[eqn_number]);
3575  }
3576  }
3577  }
3578  }
3579 
3580 
3581  /// Set history values of dofs from the type of vector stored in
3582  /// problem::Dof_pt.
3583  void Problem::set_dofs(const unsigned& t, Vector<double*>& dof_pt)
3584  {
3585 #ifdef PARANOID
3586  if (distributed())
3587  {
3588  throw OomphLibError("Not implemented for distributed problems!",
3589  OOMPH_EXCEPTION_LOCATION,
3590  OOMPH_CURRENT_FUNCTION);
3591  }
3592 #endif
3593 
3594  // If we have any spine meshes I think there might be more degrees
3595  // of freedom there. I don't use them though so I'll let someone who
3596  // knows what they are doing handle it. --David Shepherd
3597 
3598  // First deal with global data
3599  unsigned Nglobal_data = nglobal_data();
3600  for (unsigned i = 0; i < Nglobal_data; i++)
3601  {
3602  for (unsigned j = 0, nj = Global_data_pt[i]->nvalue(); j < nj; j++)
3603  {
3604  // For each data get the equation number and copy in the value.
3605  int eqn_number = Global_data_pt[i]->eqn_number(j);
3606  if (eqn_number >= 0)
3607  {
3608  Global_data_pt[i]->set_value(t, j, *(dof_pt[eqn_number]));
3609  }
3610  }
3611  }
3612 
3613  // Now the mesh data
3614  // nodes
3615  for (unsigned i = 0, ni = mesh_pt()->nnode(); i < ni; i++)
3616  {
3617  Node* node_pt = mesh_pt()->node_pt(i);
3618  for (unsigned j = 0, nj = node_pt->nvalue(); j < nj; j++)
3619  {
3620  // For each node get the equation number and copy in the value.
3621  int eqn_number = node_pt->eqn_number(j);
3622  if (eqn_number >= 0)
3623  {
3624  node_pt->set_value(t, j, *(dof_pt[eqn_number]));
3625  }
3626  }
3627  }
3628 
3629  // and non-nodal data inside elements
3630  for (unsigned i = 0, ni = mesh_pt()->nelement(); i < ni; i++)
3631  {
3632  GeneralisedElement* ele_pt = mesh_pt()->element_pt(i);
3633  for (unsigned j = 0, nj = ele_pt->ninternal_data(); j < nj; j++)
3634  {
3635  Data* data_pt = ele_pt->internal_data_pt(j);
3636  // For each node get the equation number and copy in the value.
3637  int eqn_number = data_pt->eqn_number(j);
3638  if (eqn_number >= 0)
3639  {
3640  data_pt->set_value(t, j, *(dof_pt[eqn_number]));
3641  }
3642  }
3643  }
3644  }
3645 
3646 
3647  //===================================================================
3648  /// Function that adds the values to the dofs
3649  //==================================================================
3650  void Problem::add_to_dofs(const double& lambda,
3651  const DoubleVector& increment_dofs)
3652  {
3653  const unsigned long n_dof = this->ndof();
3654  for (unsigned long l = 0; l < n_dof; l++)
3655  {
3656  *Dof_pt[l] += lambda * increment_dofs[l];
3657  }
3658  }
3659 
3660 
3661  //=========================================================================
3662  /// Return the residual vector multiplied by the inverse mass matrix
3663  /// Virtual so that it can be overloaded for mpi problems
3664  //=========================================================================
3666  {
3667  // This function does not make sense for assembly handlers other than the
3668  // default, so complain if we try to call it with another handler
3669 
3670 #ifdef PARANOID
3671  // If we are not the default, then complain
3673  {
3674  std::ostringstream error_stream;
3675  error_stream << "The function get_inverse_mass_matrix_times_residuals() "
3676  "can only be\n"
3677  << "used with the default assembly handler\n\n";
3678  throw OomphLibError(
3679  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
3680  }
3681 #endif
3682 
3683  // Find the number of degrees of freedom in the problem
3684  const unsigned n_dof = this->ndof();
3685 
3686  // Resize the vector
3687  LinearAlgebraDistribution dist(this->communicator_pt(), n_dof, false);
3688  Mres.build(&dist, 0.0);
3689 
3690  // If we have discontinuous formulation
3691  // We can invert the mass matrix element by element
3693  {
3694  // Loop over the elements and get their residuals
3695  const unsigned n_element = Problem::mesh_pt()->nelement();
3696  Vector<double> element_Mres;
3697  for (unsigned e = 0; e < n_element; e++)
3698  {
3699  // Cache the element
3700  DGElement* const elem_pt =
3701  dynamic_cast<DGElement*>(Problem::mesh_pt()->element_pt(e));
3702 
3703  // Find the elemental inverse mass matrix times residuals
3704  const unsigned n_el_dofs = elem_pt->ndof();
3705  elem_pt->get_inverse_mass_matrix_times_residuals(element_Mres);
3706 
3707  // Add contribution to global matrix
3708  for (unsigned i = 0; i < n_el_dofs; i++)
3709  {
3710  Mres[elem_pt->eqn_number(i)] = element_Mres[i];
3711  }
3712  }
3713  }
3714  // Otherwise it's continous and we must invert the full
3715  // mass matrix via a global linear solve.
3716  else
3717  {
3718  // Now do the linear solve -- recycling Mass matrix if requested
3719  // If we already have the factorised mass matrix, then resolve
3721  {
3723  {
3724  oomph_info << "Not recomputing Mass Matrix " << std::endl;
3725  }
3726 
3727  // Get the residuals
3728  DoubleVector residuals(&dist, 0.0);
3729  this->get_residuals(residuals);
3730 
3731  // Resolve the linear system
3733  residuals, Mres);
3734  }
3735  // Otherwise solve for the first time
3736  else
3737  {
3738  // If we wish to reuse the mass matrix, then enable resolve
3740  {
3742  {
3743  oomph_info << "Enabling resolve in explicit timestep" << std::endl;
3744  }
3746  ->enable_resolve();
3747  }
3748 
3749  // Use a custom assembly handler to assemble and invert the mass matrix
3750 
3751  // Store the old assembly handler
3752  AssemblyHandler* old_assembly_handler_pt = this->assembly_handler_pt();
3753  // Set the assembly handler to the explicit timestep handler
3755 
3756  // Solve the linear system
3758  Mres);
3759  // The mass matrix has now been computed
3761 
3762  // Delete the Explicit Timestep handler
3763  delete this->assembly_handler_pt();
3764  // Reset the assembly handler to the original handler
3765  this->assembly_handler_pt() = old_assembly_handler_pt;
3766  }
3767  }
3768  }
3769 
3771  {
3772  // Loop over timesteppers: make them (temporarily) steady and store their
3773  // is_steady status.
3774  unsigned n_time_steppers = this->ntime_stepper();
3775  std::vector<bool> was_steady(n_time_steppers);
3776  for (unsigned i = 0; i < n_time_steppers; i++)
3777  {
3778  was_steady[i] = time_stepper_pt(i)->is_steady();
3780  }
3781 
3782  // Calculate f using the residual/jacobian machinary.
3784 
3785  // Reset the is_steady status of all timesteppers that weren't already
3786  // steady when we came in here and reset their weights
3787  for (unsigned i = 0; i < n_time_steppers; i++)
3788  {
3789  if (!was_steady[i])
3790  {
3792  }
3793  }
3794  }
3795 
3796 
3797  //================================================================
3798  /// Get the total residuals Vector for the problem
3799  //================================================================
3801  {
3802  // Three different cases; if MPI_Helpers::MPI_has_been_initialised=true
3803  // this means MPI_Helpers::init() has been called. This could happen on a
3804  // code compiled with MPI but run serially; in this instance the
3805  // get_residuals function still works on one processor.
3806  //
3807  // Secondly, if a code has been compiled with MPI, but MPI_Helpers::init()
3808  // has not been called, then MPI_Helpers::MPI_has_been_initialised=false
3809  // and the code calls...
3810  //
3811  // Thirdly, the serial version (compiled by all, but only run when compiled
3812  // with MPI if MPI_Helpers::MPI_has_been_initialised=false
3813 
3814  // Check that the residuals has the correct number of rows if it has been
3815  // setup
3816 #ifdef PARANOID
3817  if (residuals.built())
3818  {
3819  if (residuals.distribution_pt()->nrow() != this->ndof())
3820  {
3821  std::ostringstream error_stream;
3822  error_stream << "The distribution of the residuals vector does not "
3823  "have the correct\n"
3824  << "number of global rows\n";
3825 
3826  throw OomphLibError(
3827  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
3828  }
3829  }
3830 #endif
3831 
3832  // Determine the distribution for the residuals vector
3833  // IF the vector has distribution setup then use that
3834  // ELSE determine the distribution based on the
3835  // distributed_matrix_distribution enum
3836  LinearAlgebraDistribution* dist_pt = 0;
3837  if (residuals.built())
3838  {
3839  dist_pt = new LinearAlgebraDistribution(residuals.distribution_pt());
3840  }
3841  else
3842  {
3844  }
3845 
3846  // Locally cache pointer to assembly handler
3848 
3849  // Build and zero the residuals
3850  residuals.build(dist_pt, 0.0);
3851 
3852  // Serial (or one processor case)
3853 #ifdef OOMPH_HAS_MPI
3854  if (this->communicator_pt()->nproc() == 1)
3855  {
3856 #endif // OOMPH_HAS_MPI
3857  // Loop over all the elements
3858  unsigned long Element_pt_range = Mesh_pt->nelement();
3859  for (unsigned long e = 0; e < Element_pt_range; e++)
3860  {
3861  // Get the pointer to the element
3862  GeneralisedElement* elem_pt = Mesh_pt->element_pt(e);
3863  // Find number of dofs in the element
3864  unsigned n_element_dofs = assembly_handler_pt->ndof(elem_pt);
3865  // Set up an array
3866  Vector<double> element_residuals(n_element_dofs);
3867  // Fill the array
3868  assembly_handler_pt->get_residuals(elem_pt, element_residuals);
3869  // Now loop over the dofs and assign values to global Vector
3870  for (unsigned l = 0; l < n_element_dofs; l++)
3871  {
3872  residuals[assembly_handler_pt->eqn_number(elem_pt, l)] +=
3873  element_residuals[l];
3874  }
3875  }
3876  // Otherwise parallel case
3877 #ifdef OOMPH_HAS_MPI
3878  }
3879  else
3880  {
3881  // Store the current assembly handler
3882  AssemblyHandler* const old_assembly_handler_pt = Assembly_handler_pt;
3883  // Create a new assembly handler that only assembles the residuals
3885  new ParallelResidualsHandler(old_assembly_handler_pt);
3886 
3887  // Setup memory for parallel sparse assemble
3888  // No matrix so all size zero
3889  Vector<int*> column_index;
3890  Vector<int*> row_start;
3891  Vector<double*> value;
3892  Vector<unsigned> nnz;
3893  // One set of residuals of sizer one
3894  Vector<double*> res(1);
3895 
3896  // Call the parallel sparse assemble, that should only assemble residuals
3898  dist_pt, column_index, row_start, value, nnz, res);
3899  // Fill in the residuals data
3900  residuals.set_external_values(res[0], true);
3901 
3902  // Delete new assembly handler
3903  delete Assembly_handler_pt;
3904  // Reset the assembly handler to the original
3905  Assembly_handler_pt = old_assembly_handler_pt;
3906  }
3907 #endif
3908 
3909  // Delete the distribution
3910  delete dist_pt;
3911  }
3912 
3913  //=============================================================================
3914  /// Get the fully assembled residual vector and Jacobian matrix
3915  /// in dense storage. The DoubleVector residuals returned will be
3916  /// non-distributed. If on calling this method the DoubleVector residuals is
3917  /// setup then it must be non-distributed and of the correct length.
3918  /// The matrix type DenseDoubleMatrix is not distributable and therefore
3919  /// the residual vector is also assumed to be non distributable.
3920  //=============================================================================
3922  DenseDoubleMatrix& jacobian)
3923  {
3924  // get the number of degrees of freedom
3925  unsigned n_dof = ndof();
3926 
3927 #ifdef PARANOID
3928  // PARANOID checks : if the distribution of residuals is setup then it must
3929  // must not be distributed, have the right number of rows, and the same
3930  // communicator as the problem
3931  if (residuals.built())
3932  {
3933  if (residuals.distribution_pt()->distributed())
3934  {
3935  std::ostringstream error_stream;
3936  error_stream
3937  << "If the DoubleVector residuals is setup then it must not "
3938  << "be distributed.";
3939  throw OomphLibError(
3940  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
3941  }
3942  if (residuals.distribution_pt()->nrow() != n_dof)
3943  {
3944  std::ostringstream error_stream;
3945  error_stream
3946  << "If the DoubleVector residuals is setup then it must have"
3947  << " the correct number of rows";
3948  throw OomphLibError(
3949  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
3950  }
3951  if (!(*Communicator_pt ==
3952  *residuals.distribution_pt()->communicator_pt()))
3953  {
3954  std::ostringstream error_stream;
3955  error_stream
3956  << "If the DoubleVector residuals is setup then it must have"
3957  << " the same communicator as the problem.";
3958  throw OomphLibError(
3959  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
3960  }
3961  }
3962 #endif
3963 
3964  // set the residuals distribution if it is not setup
3965  if (!residuals.built())
3966  {
3967  LinearAlgebraDistribution dist(Communicator_pt, n_dof, false);
3968  residuals.build(&dist, 0.0);
3969  }
3970  // else just zero the residuals
3971  else
3972  {
3973  residuals.initialise(0.0);
3974  }
3975 
3976  // Resize the matrices -- this cannot always be done externally
3977  // because get_jacobian exists in many different versions for
3978  // different storage formats -- resizing a CC or CR matrix doesn't
3979  // make sense.
3980 
3981  // resize the jacobian
3982  jacobian.resize(n_dof, n_dof);
3983  jacobian.initialise(0.0);
3984 
3985  // Locally cache pointer to assembly handler
3987 
3988  // Loop over all the elements
3989  unsigned long n_element = Mesh_pt->nelement();
3990  for (unsigned long e = 0; e < n_element; e++)
3991  {
3992  // Get the pointer to the element
3993  GeneralisedElement* elem_pt = Mesh_pt->element_pt(e);
3994  // Find number of dofs in the element
3995  unsigned n_element_dofs = assembly_handler_pt->ndof(elem_pt);
3996  // Set up an array
3997  Vector<double> element_residuals(n_element_dofs);
3998  // Set up a matrix
3999  DenseMatrix<double> element_jacobian(n_element_dofs);
4000  // Fill the array
4002  elem_pt, element_residuals, element_jacobian);
4003  // Now loop over the dofs and assign values to global Vector
4004  for (unsigned l = 0; l < n_element_dofs; l++)
4005  {
4006  unsigned long eqn_number = assembly_handler_pt->eqn_number(elem_pt, l);
4007  residuals[eqn_number] += element_residuals[l];
4008  for (unsigned l2 = 0; l2 < n_element_dofs; l2++)
4009  {
4010  jacobian(eqn_number, assembly_handler_pt->eqn_number(elem_pt, l2)) +=
4011  element_jacobian(l, l2);
4012  }
4013  }
4014  }
4015  }
4016 
4017  //=============================================================================
4018  /// Return the fully-assembled Jacobian and residuals for the problem,
4019  /// in the case where the Jacobian matrix is in a distributable
4020  /// row compressed storage format.
4021  /// 1. If the distribution of the jacobian and residuals is setup then, they
4022  /// will be returned with that distribution.
4023  /// Note. the jacobian and residuals must have the same distribution.
4024  /// 2. If the distribution of the jacobian and residuals are not setup then
4025  /// their distribution will computed based on:
4026  /// Distributed_problem_matrix_distribution.
4027  //=============================================================================
4029  {
4030  // Three different cases; if MPI_Helpers::MPI_has_been_initialised=true
4031  // this means MPI_Helpers::setup() has been called. This could happen on a
4032  // code compiled with MPI but run serially; in this instance the
4033  // get_residuals function still works on one processor.
4034  //
4035  // Secondly, if a code has been compiled with MPI, but MPI_Helpers::setup()
4036  // has not been called, then MPI_Helpers::MPI_has_been_initialised=false
4037  // and the code calls...
4038  //
4039  // Thirdly, the serial version (compiled by all, but only run when compiled
4040  // with MPI if MPI_Helpers::MPI_has_been_initialised=false
4041  //
4042  // The only case where an MPI code cannot run serially at present
4043  // is one where the distribute function is used (i.e. METIS is called)
4044 
4045  // Allocate storage for the matrix entries
4046  // The generalised Vector<Vector<>> structure is required
4047  // for the most general interface to sparse_assemble() which allows
4048  // the assembly of multiple matrices at once.
4049  Vector<int*> column_index(1);
4050  Vector<int*> row_start(1);
4051  Vector<double*> value(1);
4052  Vector<unsigned> nnz(1);
4053 
4054 #ifdef PARANOID
4055  // PARANOID checks that the distribution of the jacobian matches that of the
4056  // residuals (if they are setup) and that they have the right number of rows
4057  if (residuals.built() && jacobian.distribution_built())
4058  {
4059  if (!(*residuals.distribution_pt() == *jacobian.distribution_pt()))
4060  {
4061  std::ostringstream error_stream;
4062  error_stream << "The distribution of the residuals must "
4063  << "be the same as the distribution of the jacobian.";
4064  throw OomphLibError(
4065  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
4066  }
4067  if (jacobian.distribution_pt()->nrow() != this->ndof())
4068  {
4069  std::ostringstream error_stream;
4070  error_stream
4071  << "The distribution of the jacobian and residuals does not"
4072  << "have the correct number of global rows.";
4073  throw OomphLibError(
4074  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
4075  }
4076  }
4077  else if (residuals.built() != jacobian.distribution_built())
4078  {
4079  std::ostringstream error_stream;
4080  error_stream << "The distribution of the jacobian and residuals must "
4081  << "both be setup or both not setup";
4082  throw OomphLibError(
4083  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
4084  }
4085 #endif
4086 
4087 
4088  // Allocate generalised storage format for passing to sparse_assemble()
4089  Vector<double*> res(1);
4090 
4091  // determine the distribution for the jacobian.
4092  // IF the jacobian has distribution setup then use that
4093  // ELSE determine the distribution based on the
4094  // distributed_matrix_distribution enum
4095  LinearAlgebraDistribution* dist_pt = 0;
4096  if (jacobian.distribution_built())
4097  {
4098  dist_pt = new LinearAlgebraDistribution(jacobian.distribution_pt());
4099  }
4100  else
4101  {
4103  }
4104 
4105 
4106  // The matrix is in compressed row format
4107  bool compressed_row_flag = true;
4108 
4109 #ifdef OOMPH_HAS_MPI
4110  //
4111  if (Communicator_pt->nproc() == 1)
4112  {
4113 #endif
4115  column_index, row_start, value, nnz, res, compressed_row_flag);
4116  jacobian.build(dist_pt);
4117  jacobian.build_without_copy(
4118  dist_pt->nrow(), nnz[0], value[0], column_index[0], row_start[0]);
4119  residuals.build(dist_pt, 0.0);
4120  residuals.set_external_values(res[0], true);
4121 #ifdef OOMPH_HAS_MPI
4122  }
4123  else
4124  {
4125  if (dist_pt->distributed())
4126  {
4128  dist_pt, column_index, row_start, value, nnz, res);
4129  jacobian.build(dist_pt);
4130  jacobian.build_without_copy(
4131  dist_pt->nrow(), nnz[0], value[0], column_index[0], row_start[0]);
4132  residuals.build(dist_pt, 0.0);
4133  residuals.set_external_values(res[0], true);
4134  }
4135  else
4136  {
4137  LinearAlgebraDistribution* temp_dist_pt =
4138  new LinearAlgebraDistribution(Communicator_pt, dist_pt->nrow(), true);
4140  temp_dist_pt, column_index, row_start, value, nnz, res);
4141  jacobian.build(temp_dist_pt);
4142  jacobian.build_without_copy(
4143  dist_pt->nrow(), nnz[0], value[0], column_index[0], row_start[0]);
4144  jacobian.redistribute(dist_pt);
4145  residuals.build(temp_dist_pt, 0.0);
4146  residuals.set_external_values(res[0], true);
4147  residuals.redistribute(dist_pt);
4148  delete temp_dist_pt;
4149  }
4150  }
4151 #endif
4152 
4153  // clean up dist_pt and residuals_vector pt
4154  delete dist_pt;
4155  }
4156 
4157  //=============================================================================
4158  /// Return the fully-assembled Jacobian and residuals for the problem,
4159  /// in the case when the jacobian matrix is in column-compressed storage
4160  /// format.
4161  //=============================================================================
4163  {
4164  // Three different cases; if MPI_Helpers::MPI_has_been_initialised=true
4165  // this means MPI_Helpers::setup() has been called. This could happen on a
4166  // code compiled with MPI but run serially; in this instance the
4167  // get_residuals function still works on one processor.
4168  //
4169  // Secondly, if a code has been compiled with MPI, but MPI_Helpers::setup()
4170  // has not been called, then MPI_Helpers::MPI_has_been_initialised=false
4171  // and the code calls...
4172  //
4173  // Thirdly, the serial version (compiled by all, but only run when compiled
4174  // with MPI if MPI_Helpers::MPI_has_been_5Binitialised=false
4175  //
4176  // The only case where an MPI code cannot run serially at present
4177  // is one where the distribute function is used (i.e. METIS is called)
4178 
4179  // get the number of degrees of freedom
4180  unsigned n_dof = ndof();
4181 
4182 #ifdef PARANOID
4183  // PARANOID checks : if the distribution of residuals is setup then it must
4184  // must not be distributed, have the right number of rows, and the same
4185  // communicator as the problem
4186  if (residuals.built())
4187  {
4188  if (residuals.distribution_pt()->distributed())
4189  {
4190  std::ostringstream error_stream;
4191  error_stream
4192  << "If the DoubleVector residuals is setup then it must not "
4193  << "be distributed.";
4194  throw OomphLibError(
4195  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
4196  }
4197  if (residuals.distribution_pt()->nrow() != n_dof)
4198  {
4199  std::ostringstream error_stream;
4200  error_stream
4201  << "If the DoubleVector residuals is setup then it must have"
4202  << " the correct number of rows";
4203  throw OomphLibError(
4204  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
4205  }
4206  if (!(*Communicator_pt ==
4207  *residuals.distribution_pt()->communicator_pt()))
4208  {
4209  std::ostringstream error_stream;
4210  error_stream
4211  << "If the DoubleVector residuals is setup then it must have"
4212  << " the same communicator as the problem.";
4213  throw OomphLibError(
4214  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
4215  }
4216  }
4217 #endif
4218 
4219  // Allocate storage for the matrix entries
4220  // The generalised Vector<Vector<>> structure is required
4221  // for the most general interface to sparse_assemble() which allows
4222  // the assembly of multiple matrices at once.
4223  Vector<int*> row_index(1);
4224  Vector<int*> column_start(1);
4225  Vector<double*> value(1);
4226 
4227  // Allocate generalised storage format for passing to sparse_assemble()
4228  Vector<double*> res(1);
4229 
4230  // allocate storage for the number of non-zeros in each matrix
4231  Vector<unsigned> nnz(1);
4232 
4233  // The matrix is in compressed column format
4234  bool compressed_row_flag = false;
4235 
4236  // get the distribution for the residuals
4237  LinearAlgebraDistribution* dist_pt;
4238  if (!residuals.built())
4239  {
4240  dist_pt =
4241  new LinearAlgebraDistribution(Communicator_pt, this->ndof(), false);
4242  }
4243  else
4244  {
4245  dist_pt = new LinearAlgebraDistribution(residuals.distribution_pt());
4246  }
4247 
4248 #ifdef OOMPH_HAS_MPI
4249  if (communicator_pt()->nproc() == 1)
4250  {
4251 #endif
4253  row_index, column_start, value, nnz, res, compressed_row_flag);
4254  jacobian.build_without_copy(
4255  value[0], row_index[0], column_start[0], nnz[0], n_dof, n_dof);
4256  residuals.build(dist_pt, 0.0);
4257  residuals.set_external_values(res[0], true);
4258 #ifdef OOMPH_HAS_MPI
4259  }
4260  else
4261  {
4262  std::ostringstream error_stream;
4263  error_stream << "Cannot assemble a CCDoubleMatrix Jacobian on more "
4264  << "than one processor.";
4265  throw OomphLibError(
4266  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
4267  }
4268 #endif
4269 
4270  // clean up
4271  delete dist_pt;
4272  }
4273 
4274 
4275  //===================================================================
4276  /// Set all pinned values to zero.
4277  /// Used to set boundary conditions to be homogeneous in the copy
4278  /// of the problem used in adaptive bifurcation tracking
4279  /// (ALH: TEMPORARY HACK, WILL BE FIXED)
4280  //==================================================================
4282  {
4283  // NOTE THIS DOES NOT ZERO ANY SPINE DATA, but otherwise everything else
4284  // should be zeroed
4285 
4286  // Zero any pinned global Data
4287  const unsigned n_global_data = nglobal_data();
4288  for (unsigned i = 0; i < n_global_data; i++)
4289  {
4290  Data* const local_data_pt = Global_data_pt[i];
4291  const unsigned n_value = local_data_pt->nvalue();
4292  for (unsigned j = 0; j < n_value; j++)
4293  {
4294  // If the data value is pinned set the value to zero
4295  if (local_data_pt->is_pinned(j))
4296  {
4297  local_data_pt->set_value(j, 0.0);
4298  }
4299  }
4300  }
4301 
4302  // Loop over the submeshes:
4303  const unsigned n_sub_mesh = Sub_mesh_pt.size();
4304  if (n_sub_mesh == 0)
4305  {
4306  // Loop over the nodes in the element
4307  const unsigned n_node = Mesh_pt->nnode();
4308  for (unsigned n = 0; n < n_node; n++)
4309  {
4310  Node* const local_node_pt = Mesh_pt->node_pt(n);
4311  const unsigned n_value = local_node_pt->nvalue();
4312  for (unsigned j = 0; j < n_value; j++)
4313  {
4314  // If the data value is pinned set the value to zero
4315  if (local_node_pt->is_pinned(j))
4316  {
4317  local_node_pt->set_value(j, 0.0);
4318  }
4319  }
4320 
4321  // Try to cast to a solid node
4322  SolidNode* const local_solid_node_pt =
4323  dynamic_cast<SolidNode*>(local_node_pt);
4324  // If we are successful
4325  if (local_solid_node_pt)
4326  {
4327  // Find the dimension of the node
4328  const unsigned n_dim = local_solid_node_pt->ndim();
4329  // Find number of positions
4330  const unsigned n_position_type =
4331  local_solid_node_pt->nposition_type();
4332 
4333  for (unsigned k = 0; k < n_position_type; k++)
4334  {
4335  for (unsigned i = 0; i < n_dim; i++)
4336  {
4337  // If the generalised position is pinned,
4338  // set the value to zero
4339  if (local_solid_node_pt->position_is_pinned(k, i))
4340  {
4341  local_solid_node_pt->x_gen(k, i) = 0.0;
4342  }
4343  }
4344  }
4345  }
4346  }
4347 
4348  // Now loop over the element's and zero the internal data
4349  const unsigned n_element = Mesh_pt->nelement();
4350  for (unsigned e = 0; e < n_element; e++)
4351  {
4352  GeneralisedElement* const local_element_pt = Mesh_pt->element_pt(e);
4353  const unsigned n_internal = local_element_pt->ninternal_data();
4354  for (unsigned i = 0; i < n_internal; i++)
4355  {
4356  Data* const local_data_pt = local_element_pt->internal_data_pt(i);
4357  const unsigned n_value = local_data_pt->nvalue();
4358  for (unsigned j = 0; j < n_value; j++)
4359  {
4360  // If the data value is pinned set the value to zero
4361  if (local_data_pt->is_pinned(j))
4362  {
4363  local_data_pt->set_value(j, 0.0);
4364  }
4365  }
4366  }
4367  } // End of loop over elements
4368  }
4369  else
4370  {
4371  // Alternatively loop over all sub meshes
4372  for (unsigned m = 0; m < n_sub_mesh; m++)
4373  {
4374  // Loop over the nodes in the element
4375  const unsigned n_node = Sub_mesh_pt[m]->nnode();
4376  for (unsigned n = 0; n < n_node; n++)
4377  {
4378  Node* const local_node_pt = Sub_mesh_pt[m]->node_pt(n);
4379  const unsigned n_value = local_node_pt->nvalue();
4380  for (unsigned j = 0; j < n_value; j++)
4381  {
4382  // If the data value is pinned set the value to zero
4383  if (local_node_pt->is_pinned(j))
4384  {
4385  local_node_pt->set_value(j, 0.0);
4386  }
4387  }
4388 
4389  // Try to cast to a solid node
4390  SolidNode* const local_solid_node_pt =
4391  dynamic_cast<SolidNode*>(local_node_pt);
4392  // If we are successful
4393  if (local_solid_node_pt)
4394  {
4395  // Find the dimension of the node
4396  const unsigned n_dim = local_solid_node_pt->ndim();
4397  // Find number of positions
4398  const unsigned n_position_type =
4399  local_solid_node_pt->nposition_type();
4400 
4401  for (unsigned k = 0; k < n_position_type; k++)
4402  {
4403  for (unsigned i = 0; i < n_dim; i++)
4404  {
4405  // If the generalised position is pinned,
4406  // set the value to zero
4407  if (local_solid_node_pt->position_is_pinned(k, i))
4408  {
4409  local_solid_node_pt->x_gen(k, i) = 0.0;
4410  }
4411  }
4412  }
4413  }
4414  }
4415 
4416  // Now loop over the element's and zero the internal data
4417  const unsigned n_element = Sub_mesh_pt[m]->nelement();
4418  for (unsigned e = 0; e < n_element; e++)
4419  {
4420  GeneralisedElement* const local_element_pt =
4421  Sub_mesh_pt[m]->element_pt(e);
4422  const unsigned n_internal = local_element_pt->ninternal_data();
4423  for (unsigned i = 0; i < n_internal; i++)
4424  {
4425  Data* const local_data_pt = local_element_pt->internal_data_pt(i);
4426  const unsigned n_value = local_data_pt->nvalue();
4427  for (unsigned j = 0; j < n_value; j++)
4428  {
4429  // If the data value is pinned set the value to zero
4430  if (local_data_pt->is_pinned(j))
4431  {
4432  local_data_pt->set_value(j, 0.0);
4433  }
4434  }
4435  }
4436  } // End of loop over elements
4437  }
4438  }
4439  }
4440 
4441 
4442  //=====================================================================
4443  /// This is a (private) helper function that is used to assemble system
4444  /// matrices in compressed row or column format
4445  /// and compute residual vectors.
4446  /// The default action is to assemble the jacobian matrix and
4447  /// residuals for the Newton method. The action can be
4448  /// overloaded at an elemental level by changing the default
4449  /// behaviour of the function Element::get_all_vectors_and_matrices().
4450  /// column_or_row_index: Column [or row] index of given entry
4451  /// row_or_column_start: Index of first entry for given row [or column]
4452  /// value : Vector of nonzero entries
4453  /// residuals : Residual vector
4454  /// compressed_row_flag: Bool flag to indicate if storage format is
4455  /// compressed row [if false interpretation of
4456  /// arguments is as stated in square brackets].
4457  /// We provide four different assembly methods, each with different
4458  /// memory requirements/execution speeds. The method is set by
4459  /// the public flag Problem::Sparse_assembly_method.
4460  //=====================================================================
4462  Vector<int*>& column_or_row_index,
4463  Vector<int*>& row_or_column_start,
4464  Vector<double*>& value,
4465  Vector<unsigned>& nnz,
4466  Vector<double*>& residuals,
4467  bool compressed_row_flag)
4468  {
4469  // Choose the actual method
4470  switch (Sparse_assembly_method)
4471  {
4473 
4475  column_or_row_index,
4476  row_or_column_start,
4477  value,
4478  nnz,
4479  residuals,
4480  compressed_row_flag);
4481 
4482  break;
4483 
4485 
4487  column_or_row_index,
4488  row_or_column_start,
4489  value,
4490  nnz,
4491  residuals,
4492  compressed_row_flag);
4493 
4494  break;
4495 
4497 
4499  row_or_column_start,
4500  value,
4501  nnz,
4502  residuals,
4503  compressed_row_flag);
4504 
4505  break;
4506 
4508 
4510  column_or_row_index,
4511  row_or_column_start,
4512  value,
4513  nnz,
4514  residuals,
4515  compressed_row_flag);
4516 
4517  break;
4518 
4520 
4522  column_or_row_index,
4523  row_or_column_start,
4524  value,
4525  nnz,
4526  residuals,
4527  compressed_row_flag);
4528 
4529  break;
4530 
4531  default:
4532 
4533  std::ostringstream error_stream;
4534  error_stream
4535  << "Error: Incorrect value for Problem::Sparse_assembly_method"
4536  << Sparse_assembly_method << std::endl
4537  << "It should be one of the enumeration Problem::Assembly_method"
4538  << std::endl;
4539  throw OomphLibError(
4540  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
4541  }
4542  }
4543 
4544 
4545  //=====================================================================
4546  /// This is a (private) helper function that is used to assemble system
4547  /// matrices in compressed row or column format
4548  /// and compute residual vectors, using maps
4549  /// The default action is to assemble the jacobian matrix and
4550  /// residuals for the Newton method. The action can be
4551  /// overloaded at an elemental level by chaging the default
4552  /// behaviour of the function Element::get_all_vectors_and_matrices().
4553  /// column_or_row_index: Column [or row] index of given entry
4554  /// row_or_column_start: Index of first entry for given row [or column]
4555  /// value : Vector of nonzero entries
4556  /// residuals : Residual vector
4557  /// compressed_row_flag: Bool flag to indicate if storage format is
4558  /// compressed row [if false interpretation of
4559  /// arguments is as stated in square brackets].
4560  //=====================================================================
4562  Vector<int*>& column_or_row_index,
4563  Vector<int*>& row_or_column_start,
4564  Vector<double*>& value,
4565  Vector<unsigned>& nnz,
4566  Vector<double*>& residuals,
4567  bool compressed_row_flag)
4568  {
4569  // Total number of elements
4570  const unsigned long n_elements = mesh_pt()->nelement();
4571 
4572  // Default range of elements for distributed problems
4573  unsigned long el_lo = 0;
4574  unsigned long el_hi = n_elements - 1;
4575 
4576 #ifdef OOMPH_HAS_MPI
4577  // Otherwise just loop over a fraction of the elements
4578  // (This will either have been initialised in
4579  // Problem::set_default_first_and_last_element_for_assembly() or
4580  // will have been re-assigned during a previous assembly loop
4581  // Note that following the re-assignment only the entries
4582  // for the current processor are relevant.
4584  {
4585  el_lo = First_el_for_assembly[Communicator_pt->my_rank()];
4586  el_hi = Last_el_plus_one_for_assembly[Communicator_pt->my_rank()] - 1;
4587  }
4588 #endif
4589 
4590  // number of dofs
4591  unsigned ndof = this->ndof();
4592 
4593  // Find the number of vectors to be assembled
4594  const unsigned n_vector = residuals.size();
4595 
4596  // Find the number of matrices to be assembled
4597  const unsigned n_matrix = column_or_row_index.size();
4598 
4599  // Locally cache pointer to assembly handler
4601 
4602 #ifdef OOMPH_HAS_MPI
4603  bool doing_residuals = false;
4604  if (dynamic_cast<ParallelResidualsHandler*>(Assembly_handler_pt) != 0)
4605  {
4606  doing_residuals = true;
4607  }
4608 #endif
4609 
4610 // Error check dimensions
4611 #ifdef PARANOID
4612  if (row_or_column_start.size() != n_matrix)
4613  {
4614  std::ostringstream error_stream;
4615  error_stream << "Error: " << std::endl
4616  << "row_or_column_start.size() "
4617  << row_or_column_start.size() << " does not equal "
4618  << "column_or_row_index.size() "
4619  << column_or_row_index.size() << std::endl;
4620  throw OomphLibError(
4621  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
4622  }
4623 
4624  if (value.size() != n_matrix)
4625  {
4626  std::ostringstream error_stream;
4627  error_stream
4628  << "Error in Problem::sparse_assemble_row_or_column_compressed "
4629  << std::endl
4630  << "value.size() " << value.size() << " does not equal "
4631  << "column_or_row_index.size() " << column_or_row_index.size()
4632  << std::endl
4633  << std::endl
4634  << std::endl;
4635  throw OomphLibError(
4636  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
4637  }
4638 #endif
4639 
4640 
4641  // The idea behind this sparse assembly routine is to use a vector of
4642  // maps for the entries in each row or column of the complete matrix.
4643  // The key for each map is the global row or column number and
4644  // the default comparison operator for integers means that each map
4645  // is ordered by the global row or column number. Thus, we need not
4646  // sort the maps, that happens at each insertion of a new entry. The
4647  // price we pay is that for large maps, inseration is not a
4648  // cheap operation. Hash maps can be used to increase the speed, but then
4649  // the ordering is lost and we would have to sort anyway. The solution if
4650  // speed is required is to use lists, see below.
4651 
4652 
4653  // Set up a vector of vectors of maps of entries of each matrix,
4654  // indexed by either the column or row. The entries of the vector for
4655  // each matrix correspond to all the rows or columns of that matrix.
4656  // The use of the map storage
4657  // scheme, with its implicit ordering on the first index, gives
4658  // a sparse ordered list of the entries in the given row or column.
4659  Vector<Vector<std::map<unsigned, double>>> matrix_data_map(n_matrix);
4660  // Loop over the number of matrices being assembled and resize
4661  // each vector of maps to the number of rows or columns of the matrix
4662  for (unsigned m = 0; m < n_matrix; m++)
4663  {
4664  matrix_data_map[m].resize(ndof);
4665  }
4666 
4667  // Resize the residuals vectors
4668  for (unsigned v = 0; v < n_vector; v++)
4669  {
4670  residuals[v] = new double[ndof];
4671  for (unsigned i = 0; i < ndof; i++)
4672  {
4673  residuals[v][i] = 0;
4674  }
4675  }
4676 
4677 
4678 #ifdef OOMPH_HAS_MPI
4679 
4680 
4681  // Storage for assembly time for elements
4682  double t_assemble_start = 0.0;
4683 
4684  // Storage for assembly times
4685  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
4686  {
4687  Elemental_assembly_time.resize(n_elements);
4688  }
4689 
4690 #endif
4691 
4692  //----------------Assemble and populate the maps-------------------------
4693  {
4694  // Allocate local storage for the element's contribution to the
4695  // residuals vectors and system matrices of the size of the maximum
4696  // number of dofs in any element.
4697  // This means that the storage is only allocated (and deleted) once
4698  Vector<Vector<double>> el_residuals(n_vector);
4699  Vector<DenseMatrix<double>> el_jacobian(n_matrix);
4700 
4701  // Loop over the elements for this processor
4702  for (unsigned long e = el_lo; e <= el_hi; e++)
4703  {
4704 #ifdef OOMPH_HAS_MPI
4705  // Time it?
4706  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
4707  {
4708  t_assemble_start = TimingHelpers::timer();
4709  }
4710 #endif
4711 
4712  // Get the pointer to the element
4713  GeneralisedElement* elem_pt = mesh_pt()->element_pt(e);
4714 
4715 #ifdef OOMPH_HAS_MPI
4716  // Ignore halo elements
4717  if (!elem_pt->is_halo())
4718  {
4719 #endif
4720 
4721  // Find number of degrees of freedom in the element
4722  const unsigned nvar = assembly_handler_pt->ndof(elem_pt);
4723 
4724  // Resize the storage for elemental jacobian and residuals
4725  for (unsigned v = 0; v < n_vector; v++)
4726  {
4727  el_residuals[v].resize(nvar);
4728  }
4729  for (unsigned m = 0; m < n_matrix; m++)
4730  {
4731  el_jacobian[m].resize(nvar);
4732  }
4733 
4734  // Now get the residuals and jacobian for the element
4736  elem_pt, el_residuals, el_jacobian);
4737 
4738  //---------------Insert the values into the maps--------------
4739 
4740  // Loop over the first index of local variables
4741  for (unsigned i = 0; i < nvar; i++)
4742  {
4743  // Get the local equation number
4744  unsigned eqn_number = assembly_handler_pt->eqn_number(elem_pt, i);
4745 
4746  // Add the contribution to the residuals
4747  for (unsigned v = 0; v < n_vector; v++)
4748  {
4749  // Fill in each residuals vector
4750  residuals[v][eqn_number] += el_residuals[v][i];
4751  }
4752 
4753  // Now loop over the other index
4754  for (unsigned j = 0; j < nvar; j++)
4755  {
4756  // Get the number of the unknown
4757  unsigned unknown = assembly_handler_pt->eqn_number(elem_pt, j);
4758 
4759  // Loop over the matrices
4760  for (unsigned m = 0; m < n_matrix; m++)
4761  {
4762  // Get the value of the matrix at this point
4763  double value = el_jacobian[m](i, j);
4764  // Only bother to add to the map if it's non-zero
4765  if (std::fabs(value) > Numerical_zero_for_sparse_assembly)
4766  {
4767  // If it's compressed row storage, then our vector of maps
4768  // is indexed by row (equation number)
4769  if (compressed_row_flag)
4770  {
4771  // Add the data into the map using the unknown as the map
4772  // key
4773  matrix_data_map[m][eqn_number][unknown] += value;
4774  }
4775  // Otherwise it's compressed column storage and our vector is
4776  // indexed by column (the unknown)
4777  else
4778  {
4779  // Add the data into the map using the eqn_numbe as the map
4780  // key
4781  matrix_data_map[m][unknown][eqn_number] += value;
4782  }
4783  }
4784  } // End of loop over matrices
4785  }
4786  }
4787 
4788 #ifdef OOMPH_HAS_MPI
4789  } // endif halo element
4790 #endif
4791 
4792 
4793 #ifdef OOMPH_HAS_MPI
4794  // Time it?
4795  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
4796  {
4798  TimingHelpers::timer() - t_assemble_start;
4799  }
4800 #endif
4801 
4802  } // End of loop over the elements
4803 
4804  } // End of map assembly
4805 
4806 
4807 #ifdef OOMPH_HAS_MPI
4808 
4809  // Postprocess timing information and re-allocate distribution of
4810  // elements during subsequent assemblies.
4811  if ((!doing_residuals) && (!Problem_has_been_distributed) &&
4813  {
4815  }
4816 
4817  // We have determined load balancing for current setup.
4818  // This can remain the same until assign_eqn_numbers() is called
4819  // again -- the flag is re-set to true there.
4820  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
4821  {
4823  }
4824 
4825 #endif
4826 
4827 
4828  //-----------Finally we need to convert the beautiful map storage scheme
4829  //------------------------to the containers required by SuperLU
4830 
4831  // Loop over the number of matrices
4832  for (unsigned m = 0; m < n_matrix; m++)
4833  {
4834  // Set the number of rows or columns
4835  row_or_column_start[m] = new int[ndof + 1];
4836  // Counter for the total number of entries in the storage scheme
4837  unsigned long entry_count = 0;
4838  row_or_column_start[m][0] = entry_count;
4839 
4840  // first we compute the number of non-zeros
4841  nnz[m] = 0;
4842  for (unsigned long i_global = 0; i_global < ndof; i_global++)
4843  {
4844  nnz[m] += matrix_data_map[m][i_global].size();
4845  }
4846 
4847  // and then resize the storage
4848  column_or_row_index[m] = new int[nnz[m]];
4849  value[m] = new double[nnz[m]];
4850 
4851  // Now we merely loop over the number of rows or columns
4852  for (unsigned long i_global = 0; i_global < ndof; i_global++)
4853  {
4854  // Start index for the present row
4855  row_or_column_start[m][i_global] = entry_count;
4856  // If there are no entries in the map then skip the rest of the loop
4857  if (matrix_data_map[m][i_global].empty())
4858  {
4859  continue;
4860  }
4861 
4862  // Loop over all the entries in the map corresponding to the given
4863  // row or column. It will be ordered
4864 
4865  for (std::map<unsigned, double>::iterator it =
4866  matrix_data_map[m][i_global].begin();
4867  it != matrix_data_map[m][i_global].end();
4868  ++it)
4869  {
4870  // The first value is the column or row index
4871  column_or_row_index[m][entry_count] = it->first;
4872  // The second value is the actual data value
4873  value[m][entry_count] = it->second;
4874  // Increase the value of the counter
4875  entry_count++;
4876  }
4877  }
4878 
4879  // Final entry in the row/column start vector
4880  row_or_column_start[m][ndof] = entry_count;
4881  } // End of the loop over the matrices
4882 
4884  {
4885  oomph_info << "Pausing at end of sparse assembly." << std::endl;
4886  pause("Check memory usage now.");
4887  }
4888  }
4889 
4890 
4891  //=====================================================================
4892  /// This is a (private) helper function that is used to assemble system
4893  /// matrices in compressed row or column format
4894  /// and compute residual vectors using lists
4895  /// The default action is to assemble the jacobian matrix and
4896  /// residuals for the Newton method. The action can be
4897  /// overloaded at an elemental level by chaging the default
4898  /// behaviour of the function Element::get_all_vectors_and_matrices().
4899  /// column_or_row_index: Column [or row] index of given entry
4900  /// row_or_column_start: Index of first entry for given row [or column]
4901  /// value : Vector of nonzero entries
4902  /// residuals : Residual vector
4903  /// compressed_row_flag: Bool flag to indicate if storage format is
4904  /// compressed row [if false interpretation of
4905  /// arguments is as stated in square brackets].
4906  //=====================================================================
4908  Vector<int*>& column_or_row_index,
4909  Vector<int*>& row_or_column_start,
4910  Vector<double*>& value,
4911  Vector<unsigned>& nnz,
4912  Vector<double*>& residuals,
4913  bool compressed_row_flag)
4914  {
4915  // Total number of elements
4916  const unsigned long n_elements = mesh_pt()->nelement();
4917 
4918  // Default range of elements for distributed problems
4919  unsigned long el_lo = 0;
4920  unsigned long el_hi = n_elements - 1;
4921 
4922 #ifdef OOMPH_HAS_MPI
4923  // Otherwise just loop over a fraction of the elements
4924  // (This will either have been initialised in
4925  // Problem::set_default_first_and_last_element_for_assembly() or
4926  // will have been re-assigned during a previous assembly loop
4927  // Note that following the re-assignment only the entries
4928  // for the current processor are relevant.
4930  {
4931  el_lo = First_el_for_assembly[Communicator_pt->my_rank()];
4932  el_hi = Last_el_plus_one_for_assembly[Communicator_pt->my_rank()] - 1;
4933  }
4934 #endif
4935 
4936  // number of dofs
4937  unsigned ndof = this->ndof();
4938 
4939  // Find the number of vectors to be assembled
4940  const unsigned n_vector = residuals.size();
4941 
4942  // Find the number of matrices to be assembled
4943  const unsigned n_matrix = column_or_row_index.size();
4944 
4945  // Locally cache pointer to assembly handler
4947 
4948 #ifdef OOMPH_HAS_MPI
4949  bool doing_residuals = false;
4950  if (dynamic_cast<ParallelResidualsHandler*>(Assembly_handler_pt) != 0)
4951  {
4952  doing_residuals = true;
4953  }
4954 #endif
4955 
4956 // Error check dimensions
4957 #ifdef PARANOID
4958  if (row_or_column_start.size() != n_matrix)
4959  {
4960  std::ostringstream error_stream;
4961  error_stream << "Error: " << std::endl
4962  << "row_or_column_start.size() "
4963  << row_or_column_start.size() << " does not equal "
4964  << "column_or_row_index.size() "
4965  << column_or_row_index.size() << std::endl;
4966  throw OomphLibError(
4967  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
4968  }
4969 
4970  if (value.size() != n_matrix)
4971  {
4972  std::ostringstream error_stream;
4973  error_stream
4974  << "Error in Problem::sparse_assemble_row_or_column_compressed "
4975  << std::endl
4976  << "value.size() " << value.size() << " does not equal "
4977  << "column_or_row_index.size() " << column_or_row_index.size()
4978  << std::endl
4979  << std::endl
4980  << std::endl;
4981  throw OomphLibError(
4982  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
4983  }
4984 #endif
4985 
4986  // The idea behind this sparse assembly routine is to use a vector of
4987  // lists for the entries in each row or column of the complete matrix.
4988  // The lists contain pairs of entries (global row/column number, value).
4989  // All non-zero contributions from each element are added to the lists.
4990  // We then sort each list by global row/column number and then combine
4991  // the entries corresponding to each row/column before adding to the
4992  // vectors column_or_row_index and value.
4993 
4994  // Note the trade off for "fast assembly" is that we will require
4995  // more memory during the assembly phase. Then again, if we can
4996  // only just assemble the sparse matrix, we're in real trouble.
4997 
4998  // Set up a vector of lists of paired entries of
4999  //(row/column index, jacobian matrix entry).
5000  // The entries of the vector correspond to all the rows or columns.
5001  // The use of the list storage scheme, should give fast insertion
5002  // and fast sorts later.
5004  n_matrix);
5005  // Loop over the number of matrices and resize
5006  for (unsigned m = 0; m < n_matrix; m++)
5007  {
5008  matrix_data_list[m].resize(ndof);
5009  }
5010 
5011  // Resize the residuals vectors
5012  for (unsigned v = 0; v < n_vector; v++)
5013  {
5014  residuals[v] = new double[ndof];
5015  for (unsigned i = 0; i < ndof; i++)
5016  {
5017  residuals[v][i] = 0;
5018  }
5019  }
5020 
5021 #ifdef OOMPH_HAS_MPI
5022 
5023 
5024  // Storage for assembly time for elements
5025  double t_assemble_start = 0.0;
5026 
5027  // Storage for assembly times
5028  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
5029  {
5030  Elemental_assembly_time.resize(n_elements);
5031  }
5032 
5033 #endif
5034 
5035  //------------Assemble and populate the lists-----------------------
5036  {
5037  // Allocate local storage for the element's contribution to the
5038  // residuals vectors and system matrices of the size of the maximum
5039  // number of dofs in any element.
5040  // This means that the stored is only allocated (and deleted) once
5041  Vector<Vector<double>> el_residuals(n_vector);
5042  Vector<DenseMatrix<double>> el_jacobian(n_matrix);
5043 
5044 
5045  // Pointer to a single list to be used during the assembly
5046  std::list<std::pair<unsigned, double>>* list_pt;
5047 
5048  // Loop over the all elements
5049  for (unsigned long e = el_lo; e <= el_hi; e++)
5050  {
5051 #ifdef OOMPH_HAS_MPI
5052  // Time it?
5053  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
5054  {
5055  t_assemble_start = TimingHelpers::timer();
5056  }
5057 #endif
5058 
5059  // Get the pointer to the element
5060  GeneralisedElement* elem_pt = mesh_pt()->element_pt(e);
5061 
5062 #ifdef OOMPH_HAS_MPI
5063  // Ignore halo elements
5064  if (!elem_pt->is_halo())
5065  {
5066 #endif
5067 
5068  // Find number of degrees of freedom in the element
5069  const unsigned nvar = assembly_handler_pt->ndof(elem_pt);
5070 
5071  // Resize the storage for the elemental jacobian and residuals
5072  for (unsigned v = 0; v < n_vector; v++)
5073  {
5074  el_residuals[v].resize(nvar);
5075  }
5076  for (unsigned m = 0; m < n_matrix; m++)
5077  {
5078  el_jacobian[m].resize(nvar);
5079  }
5080 
5081  // Now get the residuals and jacobian for the element
5083  elem_pt, el_residuals, el_jacobian);
5084 
5085  //---------------- Insert the values into the lists -----------
5086 
5087  // Loop over the first index of local variables
5088  for (unsigned i = 0; i < nvar; i++)
5089  {
5090  // Get the local equation number
5091  unsigned eqn_number = assembly_handler_pt->eqn_number(elem_pt, i);
5092 
5093  // Add the contribution to the residuals
5094  for (unsigned v = 0; v < n_vector; v++)
5095  {
5096  // Fill in the residuals vector
5097  residuals[v][eqn_number] += el_residuals[v][i];
5098  }
5099 
5100  // Now loop over the other index
5101  for (unsigned j = 0; j < nvar; j++)
5102  {
5103  // Get the number of the unknown
5104  unsigned unknown = assembly_handler_pt->eqn_number(elem_pt, j);
5105 
5106  // Loop over the matrices
5107  for (unsigned m = 0; m < n_matrix; m++)
5108  {
5109  // Get the value of the matrix at this point
5110  double value = el_jacobian[m](i, j);
5111  // Only add to theif it's non-zero
5112  if (std::fabs(value) > Numerical_zero_for_sparse_assembly)
5113  {
5114  // If it's compressed row storage, then our vector is indexed
5115  // by row (the equation number)
5116  if (compressed_row_flag)
5117  {
5118  // Find the list that corresponds to the desired row
5119  list_pt = &matrix_data_list[m][eqn_number];
5120  // Insert the data into the list, the first entry
5121  // in the pair is the unknown (column index),
5122  // the second is the value itself.
5123  list_pt->insert(list_pt->end(),
5124  std::make_pair(unknown, value));
5125  }
5126  // Otherwise it's compressed column storage, and our
5127  // vector is indexed by column (the unknown)
5128  else
5129  {
5130  // Find the list that corresponds to the desired column
5131  list_pt = &matrix_data_list[m][unknown];
5132  // Insert the data into the list, the first entry
5133  // in the pair is the equation number (row index),
5134  // the second is the value itself.
5135  list_pt->insert(list_pt->end(),
5136  std::make_pair(eqn_number, value));
5137  }
5138  }
5139  }
5140  }
5141  }
5142 
5143 #ifdef OOMPH_HAS_MPI
5144  } // endif halo element
5145 #endif
5146 
5147 
5148 #ifdef OOMPH_HAS_MPI
5149  // Time it?
5150  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
5151  {
5153  TimingHelpers::timer() - t_assemble_start;
5154  }
5155 #endif
5156 
5157  } // End of loop over the elements
5158 
5159  } // list_pt goes out of scope
5160 
5161 
5162 #ifdef OOMPH_HAS_MPI
5163 
5164  // Postprocess timing information and re-allocate distribution of
5165  // elements during subsequent assemblies.
5166  if ((!doing_residuals) && (!Problem_has_been_distributed) &&
5168  {
5170  }
5171 
5172  // We have determined load balancing for current setup.
5173  // This can remain the same until assign_eqn_numbers() is called
5174  // again -- the flag is re-set to true there.
5175  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
5176  {
5178  }
5179 
5180 #endif
5181 
5182 
5183  //----Finally we need to convert the beautiful list storage scheme---
5184  //----------to the containers required by SuperLU--------------------
5185 
5186  // Loop over the number of matrices
5187  for (unsigned m = 0; m < n_matrix; m++)
5188  {
5189  // Set the number of rows or columns
5190  row_or_column_start[m] = new int[ndof + 1];
5191  // Counter for the total number of entries in the storage scheme
5192  unsigned long entry_count = 0;
5193  // The first entry is 0
5194  row_or_column_start[m][0] = entry_count;
5195 
5196  // first we compute the number of non-zeros
5197  nnz[m] = 0;
5198  for (unsigned long i_global = 0; i_global < ndof; i_global++)
5199  {
5200  nnz[m] += matrix_data_list[m][i_global].size();
5201  }
5202 
5203  // and then resize the storage
5204  column_or_row_index[m] = new int[nnz[m]];
5205  value[m] = new double[nnz[m]];
5206 
5207  // Now we merely loop over the number of rows or columns
5208  for (unsigned long i_global = 0; i_global < ndof; i_global++)
5209  {
5210  // Start index for the present row is the number of entries so far
5211  row_or_column_start[m][i_global] = entry_count;
5212  // If there are no entries in the list then skip the loop
5213  if (matrix_data_list[m][i_global].empty())
5214  {
5215  continue;
5216  }
5217 
5218  // Sort the list corresponding to this row or column by the
5219  // column or row index (first entry in the pair).
5220  // This might be inefficient, but we only have to do the sort ONCE
5221  // for each list. This is faster than using a map storage scheme, where
5222  // we are sorting for every insertion (although the map structure
5223  // is cleaner and more memory efficient)
5224  matrix_data_list[m][i_global].sort();
5225 
5226  // Set up an iterator for start of the list
5227  std::list<std::pair<unsigned, double>>::iterator it =
5228  matrix_data_list[m][i_global].begin();
5229 
5230  // Get the first row or column index in the list...
5231  unsigned current_index = it->first;
5232  //...and the corresponding value
5233  double current_value = it->second;
5234 
5235  // Loop over all the entries in the sorted list
5236  // Increase the iterator so that we start at the second entry
5237  for (++it; it != matrix_data_list[m][i_global].end(); ++it)
5238  {
5239  // If the index has not changed, then we must add the contribution
5240  // of the present entry to the value.
5241  // Additionally check that the entry is non-zero
5242  if ((it->first == current_index) &&
5243  (std::fabs(it->second) > Numerical_zero_for_sparse_assembly))
5244  {
5245  current_value += it->second;
5246  }
5247  // Otherwise, we have added all the contributions to the index
5248  // to current_value, so add it to the SuperLU data structure
5249  else
5250  {
5251  // Add the row or column index to the vector
5252  column_or_row_index[m][entry_count] = current_index;
5253  // Add the actual value to the vector
5254  value[m][entry_count] = current_value;
5255  // Increase the counter for the number of entries in each vector
5256  entry_count++;
5257 
5258  // Set the index and value to be those of the current entry in the
5259  // list
5260  current_index = it->first;
5261  current_value = it->second;
5262  }
5263  } // End of loop over all list entries for this global row or column
5264 
5265  // There are TWO special cases to consider.
5266  // If there is only one equation number in the list, then it
5267  // will NOT have been added. We test this case by comparing the
5268  // number of entries with those stored in row_or_column_start[i_global]
5269  // Otherwise
5270  // If the final entry in the list has the same index as the penultimate
5271  // entry, then it will NOT have been added to the SuperLU storage scheme
5272  // Check this by comparing the current_index with the final index
5273  // stored in the SuperLU scheme. If they are not the same, then
5274  // add the current_index and value.
5275 
5276  // If single equation number in list
5277  if ((static_cast<int>(entry_count) == row_or_column_start[m][i_global])
5278  // If we have a single equation number, this will not be evaluated.
5279  // If we don't then we do the test to check that the final
5280  // entry is added
5281  || (static_cast<int>(current_index) !=
5282  column_or_row_index[m][entry_count - 1]))
5283  {
5284  // Add the row or column index to the vector
5285  column_or_row_index[m][entry_count] = current_index;
5286  // Add the actual value to the vector
5287  value[m][entry_count] = current_value;
5288  // Increase the counter for the number of entries in each vector
5289  entry_count++;
5290  }
5291 
5292  } // End of loop over the rows or columns of the entire matrix
5293 
5294  // Final entry in the row/column start vector
5295  row_or_column_start[m][ndof] = entry_count;
5296  } // End of loop over matrices
5297 
5299  {
5300  oomph_info << "Pausing at end of sparse assembly." << std::endl;
5301  pause("Check memory usage now.");
5302  }
5303  }
5304 
5305 
5306  //=====================================================================
5307  /// This is a (private) helper function that is used to assemble system
5308  /// matrices in compressed row or column format
5309  /// and compute residual vectors using vectors of pairs
5310  /// The default action is to assemble the jacobian matrix and
5311  /// residuals for the Newton method. The action can be
5312  /// overloaded at an elemental level by chaging the default
5313  /// behaviour of the function Element::get_all_vectors_and_matrices().
5314  /// column_or_row_index: Column [or row] index of given entry
5315  /// row_or_column_start: Index of first entry for given row [or column]
5316  /// value : Vector of nonzero entries
5317  /// residuals : Residual vector
5318  /// compressed_row_flag: Bool flag to indicate if storage format is
5319  /// compressed row [if false interpretation of
5320  /// arguments is as stated in square brackets].
5321  //=====================================================================
5323  Vector<int*>& column_or_row_index,
5324  Vector<int*>& row_or_column_start,
5325  Vector<double*>& value,
5326  Vector<unsigned>& nnz,
5327  Vector<double*>& residuals,
5328  bool compressed_row_flag)
5329  {
5330  // Total number of elements
5331  const unsigned long n_elements = mesh_pt()->nelement();
5332 
5333  // Default range of elements for distributed problems
5334  unsigned long el_lo = 0;
5335  unsigned long el_hi = n_elements - 1;
5336 
5337 #ifdef OOMPH_HAS_MPI
5338  // Otherwise just loop over a fraction of the elements
5339  // (This will either have been initialised in
5340  // Problem::set_default_first_and_last_element_for_assembly() or
5341  // will have been re-assigned during a previous assembly loop
5342  // Note that following the re-assignment only the entries
5343  // for the current processor are relevant.
5345  {
5346  el_lo = First_el_for_assembly[Communicator_pt->my_rank()];
5347  el_hi = Last_el_plus_one_for_assembly[Communicator_pt->my_rank()] - 1;
5348  }
5349 #endif
5350 
5351  // number of local eqns
5352  unsigned ndof = this->ndof();
5353 
5354  // Find the number of vectors to be assembled
5355  const unsigned n_vector = residuals.size();
5356 
5357  // Find the number of matrices to be assembled
5358  const unsigned n_matrix = column_or_row_index.size();
5359 
5360  // Locally cache pointer to assembly handler
5362 
5363 #ifdef OOMPH_HAS_MPI
5364  bool doing_residuals = false;
5365  if (dynamic_cast<ParallelResidualsHandler*>(Assembly_handler_pt) != 0)
5366  {
5367  doing_residuals = true;
5368  }
5369 #endif
5370 
5371 // Error check dimensions
5372 #ifdef PARANOID
5373  if (row_or_column_start.size() != n_matrix)
5374  {
5375  std::ostringstream error_stream;
5376  error_stream << "Error: " << std::endl
5377  << "row_or_column_start.size() "
5378  << row_or_column_start.size() << " does not equal "
5379  << "column_or_row_index.size() "
5380  << column_or_row_index.size() << std::endl;
5381  throw OomphLibError(
5382  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
5383  }
5384 
5385  if (value.size() != n_matrix)
5386  {
5387  std::ostringstream error_stream;
5388  error_stream << "Error: " << std::endl
5389  << "value.size() " << value.size() << " does not equal "
5390  << "column_or_row_index.size() "
5391  << column_or_row_index.size() << std::endl
5392  << std::endl
5393  << std::endl;
5394  throw OomphLibError(
5395  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
5396  }
5397 #endif
5398 
5399 
5400  // The idea behind this sparse assembly routine is to use a Vector of
5401  // Vectors of pairs for each complete matrix.
5402  // Each inner Vector stores pairs and holds the row (or column) index
5403  // and the value of the matrix entry.
5404 
5405  // Set up Vector of Vectors to store the entries of each matrix,
5406  // indexed by either the column or row.
5407  Vector<Vector<Vector<std::pair<unsigned, double>>>> matrix_data(n_matrix);
5408 
5409  // Loop over the number of matrices being assembled and resize
5410  // each Vector of Vectors to the number of rows or columns of the matrix
5411  for (unsigned m = 0; m < n_matrix; m++)
5412  {
5413  matrix_data[m].resize(ndof);
5414  }
5415 
5416  // Resize the residuals vectors
5417  for (unsigned v = 0; v < n_vector; v++)
5418  {
5419  residuals[v] = new double[ndof];
5420  for (unsigned i = 0; i < ndof; i++)
5421  {
5422  residuals[v][i] = 0;
5423  }
5424  }
5425 
5426 #ifdef OOMPH_HAS_MPI
5427 
5428  // Storage for assembly time for elements
5429  double t_assemble_start = 0.0;
5430 
5431  // Storage for assembly times
5432  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
5433  {
5434  Elemental_assembly_time.resize(n_elements);
5435  }
5436 
5437 #endif
5438 
5439  //----------------Assemble and populate the vector storage scheme--------
5440  {
5441  // Allocate local storage for the element's contribution to the
5442  // residuals vectors and system matrices of the size of the maximum
5443  // number of dofs in any element
5444  // This means that the storage is only allocated (and deleted) once
5445  Vector<Vector<double>> el_residuals(n_vector);
5446  Vector<DenseMatrix<double>> el_jacobian(n_matrix);
5447 
5448  // Loop over the elements
5449  for (unsigned long e = el_lo; e <= el_hi; e++)
5450  {
5451 #ifdef OOMPH_HAS_MPI
5452  // Time it?
5453  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
5454  {
5455  t_assemble_start = TimingHelpers::timer();
5456  }
5457 #endif
5458 
5459  // Get the pointer to the element
5460  GeneralisedElement* elem_pt = mesh_pt()->element_pt(e);
5461 
5462 #ifdef OOMPH_HAS_MPI
5463  // Ignore halo elements
5464  if (!elem_pt->is_halo())
5465  {
5466 #endif
5467 
5468  // Find number of degrees of freedom in the element
5469  const unsigned nvar = assembly_handler_pt->ndof(elem_pt);
5470 
5471  // Resize the storage for elemental jacobian and residuals
5472  for (unsigned v = 0; v < n_vector; v++)
5473  {
5474  el_residuals[v].resize(nvar);
5475  }
5476  for (unsigned m = 0; m < n_matrix; m++)
5477  {
5478  el_jacobian[m].resize(nvar);
5479  }
5480 
5481  // Now get the residuals and jacobian for the element
5483  elem_pt, el_residuals, el_jacobian);
5484 
5485  //---------------Insert the values into the vectors--------------
5486 
5487  // Loop over the first index of local variables
5488  for (unsigned i = 0; i < nvar; i++)
5489  {
5490  // Get the local equation number
5491  unsigned eqn_number = assembly_handler_pt->eqn_number(elem_pt, i);
5492 
5493  // Add the contribution to the residuals
5494  for (unsigned v = 0; v < n_vector; v++)
5495  {
5496  // Fill in each residuals vector
5497  residuals[v][eqn_number] += el_residuals[v][i];
5498  }
5499 
5500  // Now loop over the other index
5501  for (unsigned j = 0; j < nvar; j++)
5502  {
5503  // Get the number of the unknown
5504  unsigned unknown = assembly_handler_pt->eqn_number(elem_pt, j);
5505 
5506  // Loop over the matrices
5507  // If it's compressed row storage, then our vector of maps
5508  // is indexed by row (equation number)
5509  for (unsigned m = 0; m < n_matrix; m++)
5510  {
5511  // Get the value of the matrix at this point
5512  double value = el_jacobian[m](i, j);
5513  // Only bother to add to the vector if it's non-zero
5514  if (std::fabs(value) > Numerical_zero_for_sparse_assembly)
5515  {
5516  // If it's compressed row storage, then our vector of maps
5517  // is indexed by row (equation number)
5518  if (compressed_row_flag)
5519  {
5520  // Find the correct position and add the data into the
5521  // vectors
5522  const unsigned size = matrix_data[m][eqn_number].size();
5523  for (unsigned k = 0; k <= size; k++)
5524  {
5525  if (k == size)
5526  {
5527  matrix_data[m][eqn_number].push_back(
5528  std::make_pair(unknown, value));
5529  break;
5530  }
5531  else if (matrix_data[m][eqn_number][k].first == unknown)
5532  {
5533  matrix_data[m][eqn_number][k].second += value;
5534  break;
5535  }
5536  }
5537  }
5538  // Otherwise it's compressed column storage and our vector is
5539  // indexed by column (the unknown)
5540  else
5541  {
5542  // Add the data into the vectors in the correct position
5543  const unsigned size = matrix_data[m][unknown].size();
5544  for (unsigned k = 0; k <= size; k++)
5545  {
5546  if (k == size)
5547  {
5548  matrix_data[m][unknown].push_back(
5549  std::make_pair(eqn_number, value));
5550  break;
5551  }
5552  else if (matrix_data[m][unknown][k].first == eqn_number)
5553  {
5554  matrix_data[m][unknown][k].second += value;
5555  break;
5556  }
5557  }
5558  }
5559  }
5560  } // End of loop over matrices
5561  }
5562  }
5563 
5564 #ifdef OOMPH_HAS_MPI
5565  } // endif halo element
5566 #endif
5567 
5568 
5569 #ifdef OOMPH_HAS_MPI
5570  // Time it?
5571  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
5572  {
5574  TimingHelpers::timer() - t_assemble_start;
5575  }
5576 #endif
5577 
5578  } // End of loop over the elements
5579 
5580 
5581  } // End of vector assembly
5582 
5583 
5584 #ifdef OOMPH_HAS_MPI
5585 
5586  // Postprocess timing information and re-allocate distribution of
5587  // elements during subsequent assemblies.
5588  if ((!doing_residuals) && (!Problem_has_been_distributed) &&
5590  {
5592  }
5593 
5594  // We have determined load balancing for current setup.
5595  // This can remain the same until assign_eqn_numbers() is called
5596  // again -- the flag is re-set to true there.
5597  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
5598  {
5600  }
5601 
5602 #endif
5603 
5604 
5605  //-----------Finally we need to convert this vector storage scheme
5606  //------------------------to the containers required by SuperLU
5607 
5608  // Loop over the number of matrices
5609  for (unsigned m = 0; m < n_matrix; m++)
5610  {
5611  // Set the number of rows or columns
5612  row_or_column_start[m] = new int[ndof + 1];
5613 
5614  // fill row_or_column_start and find the number of entries
5615  row_or_column_start[m][0] = 0;
5616  for (unsigned long i = 0; i < ndof; i++)
5617  {
5618  row_or_column_start[m][i + 1] =
5619  row_or_column_start[m][i] + matrix_data[m][i].size();
5620  }
5621  const unsigned entries = row_or_column_start[m][ndof];
5622 
5623  // resize vectors
5624  column_or_row_index[m] = new int[entries];
5625  value[m] = new double[entries];
5626  nnz[m] = entries;
5627 
5628  // Now we merely loop over the number of rows or columns
5629  for (unsigned long i_global = 0; i_global < ndof; i_global++)
5630  {
5631  // If there are no entries in the vector then skip the rest of the loop
5632  if (matrix_data[m][i_global].empty())
5633  {
5634  continue;
5635  }
5636 
5637  // Loop over all the entries in the vectors corresponding to the given
5638  // row or column. It will NOT be ordered
5639  unsigned p = 0;
5640  for (int j = row_or_column_start[m][i_global];
5641  j < row_or_column_start[m][i_global + 1];
5642  j++)
5643  {
5644  column_or_row_index[m][j] = matrix_data[m][i_global][p].first;
5645  value[m][j] = matrix_data[m][i_global][p].second;
5646  ++p;
5647  }
5648  }
5649  } // End of the loop over the matrices
5650 
5652  {
5653  oomph_info << "Pausing at end of sparse assembly." << std::endl;
5654  pause("Check memory usage now.");
5655  }
5656  }
5657 
5658 
5659  //=====================================================================
5660  /// This is a (private) helper function that is used to assemble system
5661  /// matrices in compressed row or column format
5662  /// and compute residual vectors using two vectors.
5663  /// The default action is to assemble the jacobian matrix and
5664  /// residuals for the Newton method. The action can be
5665  /// overloaded at an elemental level by chaging the default
5666  /// behaviour of the function Element::get_all_vectors_and_matrices().
5667  /// column_or_row_index: Column [or row] index of given entry
5668  /// row_or_column_start: Index of first entry for given row [or column]
5669  /// value : Vector of nonzero entries
5670  /// residuals : Residual vector
5671  /// compressed_row_flag: Bool flag to indicate if storage format is
5672  /// compressed row [if false interpretation of
5673  /// arguments is as stated in square brackets].
5674  //=====================================================================
5676  Vector<int*>& column_or_row_index,
5677  Vector<int*>& row_or_column_start,
5678  Vector<double*>& value,
5679  Vector<unsigned>& nnz,
5680  Vector<double*>& residuals,
5681  bool compressed_row_flag)
5682  {
5683  // Total number of elements
5684  const unsigned long n_elements = mesh_pt()->nelement();
5685 
5686  // Default range of elements for distributed problems
5687  unsigned long el_lo = 0;
5688  unsigned long el_hi = n_elements - 1;
5689 
5690 
5691 #ifdef OOMPH_HAS_MPI
5692  // Otherwise just loop over a fraction of the elements
5693  // (This will either have been initialised in
5694  // Problem::set_default_first_and_last_element_for_assembly() or
5695  // will have been re-assigned during a previous assembly loop
5696  // Note that following the re-assignment only the entries
5697  // for the current processor are relevant.
5699  {
5700  el_lo = First_el_for_assembly[Communicator_pt->my_rank()];
5701  el_hi = Last_el_plus_one_for_assembly[Communicator_pt->my_rank()] - 1;
5702  }
5703 #endif
5704 
5705  // number of local eqns
5706  unsigned ndof = this->ndof();
5707 
5708  // Find the number of vectors to be assembled
5709  const unsigned n_vector = residuals.size();
5710 
5711  // Find the number of matrices to be assembled
5712  const unsigned n_matrix = column_or_row_index.size();
5713 
5714  // Locally cache pointer to assembly handler
5716 
5717 #ifdef OOMPH_HAS_MPI
5718  bool doing_residuals = false;
5719  if (dynamic_cast<ParallelResidualsHandler*>(Assembly_handler_pt) != 0)
5720  {
5721  doing_residuals = true;
5722  }
5723 #endif
5724 
5725 // Error check dimensions
5726 #ifdef PARANOID
5727  if (row_or_column_start.size() != n_matrix)
5728  {
5729  std::ostringstream error_stream;
5730  error_stream << "Error: " << std::endl
5731  << "row_or_column_start.size() "
5732  << row_or_column_start.size() << " does not equal "
5733  << "column_or_row_index.size() "
5734  << column_or_row_index.size() << std::endl;
5735  throw OomphLibError(
5736  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
5737  }
5738 
5739  if (value.size() != n_matrix)
5740  {
5741  std::ostringstream error_stream;
5742  error_stream << "Error: " << std::endl
5743  << "value.size() " << value.size() << " does not equal "
5744  << "column_or_row_index.size() "
5745  << column_or_row_index.size() << std::endl
5746  << std::endl
5747  << std::endl;
5748  throw OomphLibError(
5749  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
5750  }
5751 #endif
5752 
5753  // The idea behind this sparse assembly routine is to use Vectors of
5754  // Vectors for the entries in each complete matrix. And a second
5755  // Vector of Vectors stores the global row (or column) indeces. This
5756  // will not have the memory overheads associated with the methods using
5757  // lists or maps, but insertion will be more costly.
5758 
5759  // Set up two vector of vectors to store the entries of each matrix,
5760  // indexed by either the column or row. The entries of the vector for
5761  // each matrix correspond to all the rows or columns of that matrix.
5762  Vector<Vector<Vector<unsigned>>> matrix_row_or_col_indices(n_matrix);
5763  Vector<Vector<Vector<double>>> matrix_values(n_matrix);
5764 
5765  // Loop over the number of matrices being assembled and resize
5766  // each vector of vectors to the number of rows or columns of the matrix
5767  for (unsigned m = 0; m < n_matrix; m++)
5768  {
5769  matrix_row_or_col_indices[m].resize(ndof);
5770  matrix_values[m].resize(ndof);
5771  }
5772 
5773  // Resize the residuals vectors
5774  for (unsigned v = 0; v < n_vector; v++)
5775  {
5776  residuals[v] = new double[ndof];
5777  for (unsigned i = 0; i < ndof; i++)
5778  {
5779  residuals[v][i] = 0;
5780  }
5781  }
5782 
5783 #ifdef OOMPH_HAS_MPI
5784 
5785  // Storage for assembly time for elements
5786  double t_assemble_start = 0.0;
5787 
5788  // Storage for assembly times
5789  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
5790  {
5791  Elemental_assembly_time.resize(n_elements);
5792  }
5793 
5794 #endif
5795 
5796 
5797  //----------------Assemble and populate the vector storage scheme-------
5798  {
5799  // Allocate local storage for the element's contribution to the
5800  // residuals vectors and system matrices of the size of the maximum
5801  // number of dofs in any element
5802  // This means that the storage will only be allocated (and deleted) once
5803  Vector<Vector<double>> el_residuals(n_vector);
5804  Vector<DenseMatrix<double>> el_jacobian(n_matrix);
5805 
5806  // Loop over the elements
5807  for (unsigned long e = el_lo; e <= el_hi; e++)
5808  {
5809 #ifdef OOMPH_HAS_MPI
5810  // Time it?
5811  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
5812  {
5813  t_assemble_start = TimingHelpers::timer();
5814  }
5815 #endif
5816 
5817  // Get the pointer to the element
5818  GeneralisedElement* elem_pt = mesh_pt()->element_pt(e);
5819 
5820 #ifdef OOMPH_HAS_MPI
5821  // Ignore halo elements
5822  if (!elem_pt->is_halo())
5823  {
5824 #endif
5825 
5826  // Find number of degrees of freedom in the element
5827  const unsigned nvar = assembly_handler_pt->ndof(elem_pt);
5828 
5829  // Resize the storage for elemental jacobian and residuals
5830  for (unsigned v = 0; v < n_vector; v++)
5831  {
5832  el_residuals[v].resize(nvar);
5833  }
5834  for (unsigned m = 0; m < n_matrix; m++)
5835  {
5836  el_jacobian[m].resize(nvar);
5837  }
5838 
5839  // Now get the residuals and jacobian for the element
5841  elem_pt, el_residuals, el_jacobian);
5842 
5843  //---------------Insert the values into the vectors--------------
5844 
5845  // Loop over the first index of local variables
5846  for (unsigned i = 0; i < nvar; i++)
5847  {
5848  // Get the local equation number
5849  unsigned eqn_number = assembly_handler_pt->eqn_number(elem_pt, i);
5850 
5851  // Add the contribution to the residuals
5852  for (unsigned v = 0; v < n_vector; v++)
5853  {
5854  // Fill in each residuals vector
5855  residuals[v][eqn_number] += el_residuals[v][i];
5856  }
5857 
5858  // Now loop over the other index
5859  for (unsigned j = 0; j < nvar; j++)
5860  {
5861  // Get the number of the unknown
5862  unsigned unknown = assembly_handler_pt->eqn_number(elem_pt, j);
5863 
5864  // Loop over the matrices
5865  // If it's compressed row storage, then our vector of maps
5866  // is indexed by row (equation number)
5867  for (unsigned m = 0; m < n_matrix; m++)
5868  {
5869  // Get the value of the matrix at this point
5870  double value = el_jacobian[m](i, j);
5871  // Only bother to add to the vector if it's non-zero
5872  if (std::fabs(value) > Numerical_zero_for_sparse_assembly)
5873  {
5874  // If it's compressed row storage, then our vector of maps
5875  // is indexed by row (equation number)
5876  if (compressed_row_flag)
5877  {
5878  // Find the correct position and add the data into the
5879  // vectors
5880  const unsigned size =
5881  matrix_row_or_col_indices[m][eqn_number].size();
5882 
5883  for (unsigned k = 0; k <= size; k++)
5884  {
5885  if (k == size)
5886  {
5887  matrix_row_or_col_indices[m][eqn_number].push_back(
5888  unknown);
5889  matrix_values[m][eqn_number].push_back(value);
5890  break;
5891  }
5892  else if (matrix_row_or_col_indices[m][eqn_number][k] ==
5893  unknown)
5894  {
5895  matrix_values[m][eqn_number][k] += value;
5896  break;
5897  }
5898  }
5899  }
5900  // Otherwise it's compressed column storage and our vector is
5901  // indexed by column (the unknown)
5902  else
5903  {
5904  // Add the data into the vectors in the correct position
5905  const unsigned size =
5906  matrix_row_or_col_indices[m][unknown].size();
5907  for (unsigned k = 0; k <= size; k++)
5908  {
5909  if (k == size)
5910  {
5911  matrix_row_or_col_indices[m][unknown].push_back(
5912  eqn_number);
5913  matrix_values[m][unknown].push_back(value);
5914  break;
5915  }
5916  else if (matrix_row_or_col_indices[m][unknown][k] ==
5917  eqn_number)
5918  {
5919  matrix_values[m][unknown][k] += value;
5920  break;
5921  }
5922  }
5923  }
5924  }
5925  } // End of loop over matrices
5926  }
5927  }
5928 
5929 #ifdef OOMPH_HAS_MPI
5930  } // endif halo element
5931 #endif
5932 
5933 
5934 #ifdef OOMPH_HAS_MPI
5935  // Time it?
5936  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
5937  {
5939  TimingHelpers::timer() - t_assemble_start;
5940  }
5941 #endif
5942 
5943  } // End of loop over the elements
5944 
5945  } // End of vector assembly
5946 
5947 
5948 #ifdef OOMPH_HAS_MPI
5949 
5950  // Postprocess timing information and re-allocate distribution of
5951  // elements during subsequent assemblies.
5952  if ((!doing_residuals) && (!Problem_has_been_distributed) &&
5954  {
5956  }
5957 
5958  // We have determined load balancing for current setup.
5959  // This can remain the same until assign_eqn_numbers() is called
5960  // again -- the flag is re-set to true there.
5961  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
5962  {
5964  }
5965 
5966 #endif
5967 
5968  //-----------Finally we need to convert this lousy vector storage scheme
5969  //------------------------to the containers required by SuperLU
5970 
5971  // Loop over the number of matrices
5972  for (unsigned m = 0; m < n_matrix; m++)
5973  {
5974  // Set the number of rows or columns
5975  row_or_column_start[m] = new int[ndof + 1];
5976 
5977  // fill row_or_column_start and find the number of entries
5978  row_or_column_start[m][0] = 0;
5979  for (unsigned long i = 0; i < ndof; i++)
5980  {
5981  row_or_column_start[m][i + 1] =
5982  row_or_column_start[m][i] + matrix_values[m][i].size();
5983  }
5984  const unsigned entries = row_or_column_start[m][ndof];
5985 
5986  // resize vectors
5987  column_or_row_index[m] = new int[entries];
5988  value[m] = new double[entries];
5989  nnz[m] = entries;
5990 
5991  // Now we merely loop over the number of rows or columns
5992  for (unsigned long i_global = 0; i_global < ndof; i_global++)
5993  {
5994  // If there are no entries in the vector then skip the rest of the loop
5995  if (matrix_values[m][i_global].empty())
5996  {
5997  continue;
5998  }
5999 
6000  // Loop over all the entries in the vectors corresponding to the given
6001  // row or column. It will NOT be ordered
6002  unsigned p = 0;
6003  for (int j = row_or_column_start[m][i_global];
6004  j < row_or_column_start[m][i_global + 1];
6005  j++)
6006  {
6007  column_or_row_index[m][j] = matrix_row_or_col_indices[m][i_global][p];
6008  value[m][j] = matrix_values[m][i_global][p];
6009  ++p;
6010  }
6011  }
6012  } // End of the loop over the matrices
6013 
6015  {
6016  oomph_info << "Pausing at end of sparse assembly." << std::endl;
6017  pause("Check memory usage now.");
6018  }
6019  }
6020 
6021 
6022  //=====================================================================
6023  /// This is a (private) helper function that is used to assemble system
6024  /// matrices in compressed row or column format
6025  /// and compute residual vectors using two vectors.
6026  /// The default action is to assemble the jacobian matrix and
6027  /// residuals for the Newton method. The action can be
6028  /// overloaded at an elemental level by chaging the default
6029  /// behaviour of the function Element::get_all_vectors_and_matrices().
6030  /// column_or_row_index: Column [or row] index of given entry
6031  /// row_or_column_start: Index of first entry for given row [or column]
6032  /// value : Vector of nonzero entries
6033  /// residuals : Residual vector
6034  /// compressed_row_flag: Bool flag to indicate if storage format is
6035  /// compressed row [if false interpretation of
6036  /// arguments is as stated in square brackets].
6037  //=====================================================================
6039  Vector<int*>& column_or_row_index,
6040  Vector<int*>& row_or_column_start,
6041  Vector<double*>& value,
6042  Vector<unsigned>& nnz,
6043  Vector<double*>& residuals,
6044  bool compressed_row_flag)
6045  {
6046  // Total number of elements
6047  const unsigned long n_elements = mesh_pt()->nelement();
6048 
6049  // Default range of elements for distributed problems
6050  unsigned long el_lo = 0;
6051  unsigned long el_hi = n_elements - 1;
6052 
6053 
6054 #ifdef OOMPH_HAS_MPI
6055  // Otherwise just loop over a fraction of the elements
6056  // (This will either have been initialised in
6057  // Problem::set_default_first_and_last_element_for_assembly() or
6058  // will have been re-assigned during a previous assembly loop
6059  // Note that following the re-assignment only the entries
6060  // for the current processor are relevant.
6062  {
6063  el_lo = First_el_for_assembly[Communicator_pt->my_rank()];
6064  el_hi = Last_el_plus_one_for_assembly[Communicator_pt->my_rank()] - 1;
6065  }
6066 #endif
6067 
6068  // number of local eqns
6069  unsigned ndof = this->ndof();
6070 
6071  // Find the number of vectors to be assembled
6072  const unsigned n_vector = residuals.size();
6073 
6074  // Find the number of matrices to be assembled
6075  const unsigned n_matrix = column_or_row_index.size();
6076 
6077  // Locally cache pointer to assembly handler
6079 
6080 #ifdef OOMPH_HAS_MPI
6081  bool doing_residuals = false;
6082  if (dynamic_cast<ParallelResidualsHandler*>(Assembly_handler_pt) != 0)
6083  {
6084  doing_residuals = true;
6085  }
6086 #endif
6087 
6088 // Error check dimensions
6089 #ifdef PARANOID
6090  if (row_or_column_start.size() != n_matrix)
6091  {
6092  std::ostringstream error_stream;
6093  error_stream << "Error: " << std::endl
6094  << "row_or_column_start.size() "
6095  << row_or_column_start.size() << " does not equal "
6096  << "column_or_row_index.size() "
6097  << column_or_row_index.size() << std::endl;
6098  throw OomphLibError(
6099  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
6100  }
6101 
6102  if (value.size() != n_matrix)
6103  {
6104  std::ostringstream error_stream;
6105  error_stream << "Error: " << std::endl
6106  << "value.size() " << value.size() << " does not equal "
6107  << "column_or_row_index.size() "
6108  << column_or_row_index.size() << std::endl
6109  << std::endl
6110  << std::endl;
6111  throw OomphLibError(
6112  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
6113  }
6114 #endif
6115 
6116  // The idea behind this sparse assembly routine is to use Vectors of
6117  // Vectors for the entries in each complete matrix. And a second
6118  // Vector of Vectors stores the global row (or column) indeces. This
6119  // will not have the memory overheads associated with the methods using
6120  // lists or maps, but insertion will be more costly.
6121 
6122  // Set up two vector of vectors to store the entries of each matrix,
6123  // indexed by either the column or row. The entries of the vector for
6124  // each matrix correspond to all the rows or columns of that matrix.
6125  Vector<unsigned**> matrix_row_or_col_indices(n_matrix);
6126  Vector<double**> matrix_values(n_matrix);
6127 
6128  // Loop over the number of matrices being assembled and resize
6129  // each vector of vectors to the number of rows or columns of the matrix
6130  for (unsigned m = 0; m < n_matrix; m++)
6131  {
6132  matrix_row_or_col_indices[m] = new unsigned*[ndof];
6133  matrix_values[m] = new double*[ndof];
6134  }
6135 
6136  // Resize the residuals vectors
6137  for (unsigned v = 0; v < n_vector; v++)
6138  {
6139  residuals[v] = new double[ndof];
6140  for (unsigned i = 0; i < ndof; i++)
6141  {
6142  residuals[v][i] = 0;
6143  }
6144  }
6145 
6146 #ifdef OOMPH_HAS_MPI
6147 
6148  // Storage for assembly time for elements
6149  double t_assemble_start = 0.0;
6150 
6151  // Storage for assembly times
6152  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
6153  {
6154  Elemental_assembly_time.resize(n_elements);
6155  }
6156 
6157 #endif
6158 
6159  // number of coefficients in each row
6160  Vector<Vector<unsigned>> ncoef(n_matrix);
6161  for (unsigned m = 0; m < n_matrix; m++)
6162  {
6163  ncoef[m].resize(ndof, 0);
6164  }
6165 
6167  {
6169  for (unsigned m = 0; m < n_matrix; m++)
6170  {
6172  }
6173  }
6174 
6175  //----------------Assemble and populate the vector storage scheme-------
6176  {
6177  // Allocate local storage for the element's contribution to the
6178  // residuals vectors and system matrices of the size of the maximum
6179  // number of dofs in any element
6180  // This means that the storage will only be allocated (and deleted) once
6181  Vector<Vector<double>> el_residuals(n_vector);
6182  Vector<DenseMatrix<double>> el_jacobian(n_matrix);
6183 
6184  // Loop over the elements
6185  for (unsigned long e = el_lo; e <= el_hi; e++)
6186  {
6187 #ifdef OOMPH_HAS_MPI
6188  // Time it?
6189  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
6190  {
6191  t_assemble_start = TimingHelpers::timer();
6192  }
6193 #endif
6194 
6195  // Get the pointer to the element
6196  GeneralisedElement* elem_pt = mesh_pt()->element_pt(e);
6197 
6198 #ifdef OOMPH_HAS_MPI
6199  // Ignore halo elements
6200  if (!elem_pt->is_halo())
6201  {
6202 #endif
6203 
6204  // Find number of degrees of freedom in the element
6205  const unsigned nvar = assembly_handler_pt->ndof(elem_pt);
6206 
6207  // Resize the storage for elemental jacobian and residuals
6208  for (unsigned v = 0; v < n_vector; v++)
6209  {
6210  el_residuals[v].resize(nvar);
6211  }
6212  for (unsigned m = 0; m < n_matrix; m++)
6213  {
6214  el_jacobian[m].resize(nvar);
6215  }
6216 
6217  // Now get the residuals and jacobian for the element
6219  elem_pt, el_residuals, el_jacobian);
6220 
6221  //---------------Insert the values into the vectors--------------
6222 
6223  // Loop over the first index of local variables
6224  for (unsigned i = 0; i < nvar; i++)
6225  {
6226  // Get the local equation number
6227  unsigned eqn_number = assembly_handler_pt->eqn_number(elem_pt, i);
6228 
6229  // Add the contribution to the residuals
6230  for (unsigned v = 0; v < n_vector; v++)
6231  {
6232  // Fill in each residuals vector
6233  residuals[v][eqn_number] += el_residuals[v][i];
6234  }
6235 
6236  // Now loop over the other index
6237  for (unsigned j = 0; j < nvar; j++)
6238  {
6239  // Get the number of the unknown
6240  unsigned unknown = assembly_handler_pt->eqn_number(elem_pt, j);
6241 
6242  // Loop over the matrices
6243  // If it's compressed row storage, then our vector of maps
6244  // is indexed by row (equation number)
6245  for (unsigned m = 0; m < n_matrix; m++)
6246  {
6247  // Get the value of the matrix at this point
6248  double value = el_jacobian[m](i, j);
6249  // Only bother to add to the vector if it's non-zero
6250  if (std::fabs(value) > Numerical_zero_for_sparse_assembly)
6251  {
6252  // number of entrys in this row
6253  const unsigned size = ncoef[m][eqn_number];
6254 
6255  // if no data has been allocated for this row then allocate
6256  if (size == 0)
6257  {
6258  // do we have previous allocation data
6260  [m][eqn_number] != 0)
6261  {
6262  matrix_row_or_col_indices[m][eqn_number] = new unsigned
6264  [m][eqn_number]];
6265  matrix_values[m][eqn_number] = new double
6267  [m][eqn_number]];
6268  }
6269  else
6270  {
6271  matrix_row_or_col_indices[m][eqn_number] = new unsigned
6273  matrix_values[m][eqn_number] = new double
6276  [m][eqn_number] =
6278  }
6279  }
6280 
6281  // If it's compressed row storage, then our vector of maps
6282  // is indexed by row (equation number)
6283  if (compressed_row_flag)
6284  {
6285  // next add the data
6286  for (unsigned k = 0; k <= size; k++)
6287  {
6288  if (k == size)
6289  {
6290  // do we need to allocate more storage
6292  [m][eqn_number] == ncoef[m][eqn_number])
6293  {
6294  unsigned new_allocation =
6295  ncoef[m][eqn_number] +
6297  double* new_values = new double[new_allocation];
6298  unsigned* new_indices = new unsigned[new_allocation];
6299  for (unsigned c = 0; c < ncoef[m][eqn_number]; c++)
6300  {
6301  new_values[c] = matrix_values[m][eqn_number][c];
6302  new_indices[c] =
6303  matrix_row_or_col_indices[m][eqn_number][c];
6304  }
6305  delete[] matrix_values[m][eqn_number];
6306  delete[] matrix_row_or_col_indices[m][eqn_number];
6307  matrix_values[m][eqn_number] = new_values;
6308  matrix_row_or_col_indices[m][eqn_number] =
6309  new_indices;
6311  [m][eqn_number] = new_allocation;
6312  }
6313  // and now add the data
6314  unsigned entry = ncoef[m][eqn_number];
6315  ncoef[m][eqn_number]++;
6316  matrix_row_or_col_indices[m][eqn_number][entry] =
6317  unknown;
6318  matrix_values[m][eqn_number][entry] = value;
6319  break;
6320  }
6321  else if (matrix_row_or_col_indices[m][eqn_number][k] ==
6322  unknown)
6323  {
6324  matrix_values[m][eqn_number][k] += value;
6325  break;
6326  }
6327  }
6328  }
6329  // Otherwise it's compressed column storage and our vector is
6330  // indexed by column (the unknown)
6331  else
6332  {
6333  // Add the data into the vectors in the correct position
6334  for (unsigned k = 0; k <= size; k++)
6335  {
6336  if (k == size)
6337  {
6338  // do we need to allocate more storage
6340  [m][unknown] == ncoef[m][unknown])
6341  {
6342  unsigned new_allocation =
6343  ncoef[m][unknown] +
6345  double* new_values = new double[new_allocation];
6346  unsigned* new_indices = new unsigned[new_allocation];
6347  for (unsigned c = 0; c < ncoef[m][unknown]; c++)
6348  {
6349  new_values[c] = matrix_values[m][unknown][c];
6350  new_indices[c] =
6351  matrix_row_or_col_indices[m][unknown][c];
6352  }
6353  delete[] matrix_values[m][unknown];
6354  delete[] matrix_row_or_col_indices[m][unknown];
6356  [m][unknown] = new_allocation;
6357  }
6358  // and now add the data
6359  unsigned entry = ncoef[m][unknown];
6360  ncoef[m][unknown]++;
6361  matrix_row_or_col_indices[m][unknown][entry] =
6362  eqn_number;
6363  matrix_values[m][unknown][entry] = value;
6364  break;
6365  }
6366  else if (matrix_row_or_col_indices[m][unknown][k] ==
6367  eqn_number)
6368  {
6369  matrix_values[m][unknown][k] += value;
6370  break;
6371  }
6372  }
6373  }
6374  }
6375  } // End of loop over matrices
6376  }
6377  }
6378 
6379 #ifdef OOMPH_HAS_MPI
6380  } // endif halo element
6381 #endif
6382 
6383 
6384 #ifdef OOMPH_HAS_MPI
6385  // Time it?
6386  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
6387  {
6389  TimingHelpers::timer() - t_assemble_start;
6390  }
6391 #endif
6392 
6393  } // End of loop over the elements
6394 
6395  } // End of vector assembly
6396 
6397 
6398 #ifdef OOMPH_HAS_MPI
6399 
6400  // Postprocess timing information and re-allocate distribution of
6401  // elements during subsequent assemblies.
6402  if ((!doing_residuals) && (!Problem_has_been_distributed) &&
6404  {
6406  }
6407 
6408  // We have determined load balancing for current setup.
6409  // This can remain the same until assign_eqn_numbers() is called
6410  // again -- the flag is re-set to true there.
6411  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
6412  {
6414  }
6415 
6416 #endif
6417 
6418  //-----------Finally we need to convert this lousy vector storage scheme
6419  //------------------------to the containers required by SuperLU
6420 
6421  // Loop over the number of matrices
6422  for (unsigned m = 0; m < n_matrix; m++)
6423  {
6424  // Set the number of rows or columns
6425  row_or_column_start[m] = new int[ndof + 1];
6426 
6427  // fill row_or_column_start and find the number of entries
6428  row_or_column_start[m][0] = 0;
6429  for (unsigned long i = 0; i < ndof; i++)
6430  {
6431  row_or_column_start[m][i + 1] = row_or_column_start[m][i] + ncoef[m][i];
6433  }
6434  const unsigned entries = row_or_column_start[m][ndof];
6435 
6436  // resize vectors
6437  column_or_row_index[m] = new int[entries];
6438  value[m] = new double[entries];
6439  nnz[m] = entries;
6440 
6441  // Now we merely loop over the number of rows or columns
6442  for (unsigned long i_global = 0; i_global < ndof; i_global++)
6443  {
6444  // If there are no entries in the vector then skip the rest of the loop
6445  if (ncoef[m][i_global] == 0)
6446  {
6447  continue;
6448  }
6449 
6450  // Loop over all the entries in the vectors corresponding to the given
6451  // row or column. It will NOT be ordered
6452  unsigned p = 0;
6453  for (int j = row_or_column_start[m][i_global];
6454  j < row_or_column_start[m][i_global + 1];
6455  j++)
6456  {
6457  column_or_row_index[m][j] = matrix_row_or_col_indices[m][i_global][p];
6458  value[m][j] = matrix_values[m][i_global][p];
6459  ++p;
6460  }
6461 
6462  // and delete
6463  delete[] matrix_row_or_col_indices[m][i_global];
6464  delete[] matrix_values[m][i_global];
6465  }
6466 
6467  //
6468  delete[] matrix_row_or_col_indices[m];
6469  delete[] matrix_values[m];
6470  } // End of the loop over the matrices
6471 
6473  {
6474  oomph_info << "Pausing at end of sparse assembly." << std::endl;
6475  pause("Check memory usage now.");
6476  }
6477  }
6478 
6479 
6480 #ifdef OOMPH_HAS_MPI
6481  //=======================================================================
6482  /// Helper method that returns the global equations to which
6483  /// the elements in the range el_lo to el_hi contribute on this
6484  /// processor
6485  //=======================================================================
6486  void Problem::get_my_eqns(AssemblyHandler* const& assembly_handler_pt,
6487  const unsigned& el_lo,
6488  const unsigned& el_hi,
6489  Vector<unsigned>& my_eqns)
6490  {
6491  // Index to keep track of the equations counted
6492  unsigned my_eqns_index = 0;
6493 
6494  // Loop over the selection of elements
6495  for (unsigned long e = el_lo; e <= el_hi; e++)
6496  {
6497  // Get the pointer to the element
6498  GeneralisedElement* elem_pt = this->mesh_pt()->element_pt(e);
6499 
6500  // Ignore halo elements
6501  if (!elem_pt->is_halo())
6502  {
6503  // Find number of degrees of freedom in the element
6504  const unsigned nvar = assembly_handler_pt->ndof(elem_pt);
6505  // Add the number of dofs to the current size of my_eqns
6506  my_eqns.resize(my_eqns_index + nvar);
6507 
6508  // Loop over the first index of local variables
6509  for (unsigned i = 0; i < nvar; i++)
6510  {
6511  // Get the local equation number
6512  unsigned global_eqn_number =
6513  assembly_handler_pt->eqn_number(elem_pt, i);
6514  // Add into the vector
6515  my_eqns[my_eqns_index + i] = global_eqn_number;
6516  }
6517  // Update the number of elements in the vector
6518  my_eqns_index += nvar;
6519  }
6520  }
6521 
6522  // now sort and remove duplicate entries in the vector
6523  std::sort(my_eqns.begin(), my_eqns.end());
6524  Vector<unsigned>::iterator it = std::unique(my_eqns.begin(), my_eqns.end());
6525  my_eqns.resize(it - my_eqns.begin());
6526  }
6527 
6528 
6529  //=============================================================================
6530  /// Helper method to assemble CRDoubleMatrices from distributed
6531  /// on multiple processors.
6532  //=============================================================================
6534  const LinearAlgebraDistribution* const& target_dist_pt,
6535  Vector<int*>& column_indices,
6536  Vector<int*>& row_start,
6537  Vector<double*>& values,
6538  Vector<unsigned>& nnz,
6539  Vector<double*>& residuals)
6540  {
6541  // Time assembly
6542  double t_start = TimingHelpers::timer();
6543 
6544  // my rank and nproc
6545  unsigned my_rank = Communicator_pt->my_rank();
6546  unsigned nproc = Communicator_pt->nproc();
6547 
6548  // Total number of elements
6549  const unsigned long n_elements = mesh_pt()->nelement();
6550 
6551 #ifdef PARANOID
6552  // No elements? This is usually a sign that the problem distribution has
6553  // led to one processor not having any elements. Either
6554  // a sign of something having gone wrong or a relatively small
6555  // problem on a huge number of processors
6556  if (n_elements == 0)
6557  {
6558  std::ostringstream error_stream;
6559  error_stream << "Processsor " << my_rank << " has no elements. \n"
6560  << "This is usually a sign that the problem distribution \n"
6561  << "or the load balancing have gone wrong.";
6562  OomphLibWarning(error_stream.str(),
6563  "Problem::parallel_sparse_assemble()",
6564  OOMPH_EXCEPTION_LOCATION);
6565  }
6566 #endif
6567 
6568 
6569  // Default range of elements for distributed problems.
6570  unsigned long el_lo = 0;
6571  unsigned long el_hi_plus_one = n_elements;
6572 
6573  // Otherwise just loop over a fraction of the elements
6574  // (This will either have been initialised in
6575  // Problem::set_default_first_and_last_element_for_assembly() or
6576  // will have been re-assigned during a previous assembly loop
6577  // Note that following the re-assignment only the entries
6578  // for the current processor are relevant.
6580  {
6581  el_lo = First_el_for_assembly[my_rank];
6582  el_hi_plus_one = Last_el_plus_one_for_assembly[my_rank];
6583  }
6584 
6585  // Find the number of vectors to be assembled
6586  const unsigned n_vector = residuals.size();
6587 
6588  // Find the number of matrices to be assembled
6589  const unsigned n_matrix = column_indices.size();
6590 
6591  // Locally cache pointer to assembly handler
6593 
6594  bool doing_residuals = false;
6595  if (dynamic_cast<ParallelResidualsHandler*>(Assembly_handler_pt) != 0)
6596  {
6597  doing_residuals = true;
6598  }
6599 
6600 // Error check dimensions
6601 #ifdef PARANOID
6602  if (row_start.size() != n_matrix)
6603  {
6604  std::ostringstream error_stream;
6605  error_stream << "Error: " << std::endl
6606  << "row_or_column_start.size() " << row_start.size()
6607  << " does not equal "
6608  << "column_or_row_index.size() " << column_indices.size()
6609  << std::endl;
6610  throw OomphLibError(
6611  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
6612  }
6613 
6614  if (values.size() != n_matrix)
6615  {
6616  std::ostringstream error_stream;
6617  error_stream << "Error: " << std::endl
6618  << "value.size() " << values.size() << " does not equal "
6619  << "column_or_row_index.size() " << column_indices.size()
6620  << std::endl
6621  << std::endl
6622  << std::endl;
6623  throw OomphLibError(
6624  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
6625  }
6626 #endif
6627 
6628 
6629  // start by assembling the sorted set of equations to which this processor
6630  // contributes. Essentially this is every global equation that features in
6631  // all the non-halo elements. This may not be the same as the locally-stored
6632  // dofs because some of the Nodes in non-halo elements may actually
6633  // be halos.
6634  //======================================================================
6635  Vector<unsigned> my_eqns;
6636  if (n_elements != 0)
6637  {
6638  this->get_my_eqns(
6639  assembly_handler_pt, el_lo, el_hi_plus_one - 1, my_eqns);
6640  }
6641 
6642  // number of equations
6643  unsigned my_n_eqn = my_eqns.size();
6644 
6645  // next we assemble the data into an array of arrays
6646  // =================================================
6647  // The idea behind this sparse assembly routine is to use an array of
6648  // arrays for the entries in each complete matrix. And a second
6649  // array of arrays stores the global row (or column) indeces.
6650 
6651  // Set up two vector of vectors to store the entries of each matrix,
6652  // indexed by either the column or row. The entries of the vector for
6653  // each matrix correspond to all the rows or columns of that matrix.
6654  Vector<unsigned**> matrix_col_indices(n_matrix);
6655  Vector<double**> matrix_values(n_matrix);
6656 
6657  // Loop over the number of matrices being assembled and resize
6658  // each vector of vectors to the number of rows or columns of the matrix
6659  for (unsigned m = 0; m < n_matrix; m++)
6660  {
6661  matrix_col_indices[m] = new unsigned*[my_n_eqn];
6662  matrix_values[m] = new double*[my_n_eqn];
6663  for (unsigned i = 0; i < my_n_eqn; i++)
6664  {
6665  matrix_col_indices[m][i] = 0;
6666  matrix_values[m][i] = 0;
6667  }
6668  }
6669 
6670  // Resize the residuals vectors
6671  Vector<double*> residuals_data(n_vector);
6672  for (unsigned v = 0; v < n_vector; v++)
6673  {
6674  residuals_data[v] = new double[my_n_eqn];
6675  for (unsigned i = 0; i < my_n_eqn; i++)
6676  {
6677  residuals_data[v][i] = 0;
6678  }
6679  }
6680 
6681  // Storage for assembly time for elements
6682  double t_assemble_start = 0.0;
6683 
6684  // Storage for assembly times
6685  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
6686  {
6687  Elemental_assembly_time.resize(n_elements);
6688  }
6689 
6690  // number of coefficients in each row
6691  Vector<Vector<unsigned>> ncoef(n_matrix);
6692  for (unsigned m = 0; m < n_matrix; m++)
6693  {
6694  ncoef[m].resize(my_n_eqn, 0);
6695  }
6696 
6697  // Sparse_assemble_with_arrays_previous_allocation stores the number of
6698  // coefs in each row.
6699  // if a matrix of this size has not been assembled before then resize this
6700  // storage
6702  {
6704  for (unsigned m = 0; m < n_matrix; m++)
6705  {
6706  Sparse_assemble_with_arrays_previous_allocation[m].resize(my_n_eqn, 0);
6707  }
6708  }
6709 
6710 
6711  // assemble and populate an array based storage scheme
6712  {
6713  // Allocate local storage for the element's contribution to the
6714  // residuals vectors and system matrices of the size of the maximum
6715  // number of dofs in any element
6716  // This means that the storage will only be allocated (and deleted) once
6717  Vector<Vector<double>> el_residuals(n_vector);
6718  Vector<DenseMatrix<double>> el_jacobian(n_matrix);
6719 
6720  // Loop over the elements
6721  for (unsigned long e = el_lo; e < el_hi_plus_one; e++)
6722  {
6723  // Time it?
6724  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
6725  {
6726  t_assemble_start = TimingHelpers::timer();
6727  }
6728 
6729  // Get the pointer to the element
6730  GeneralisedElement* elem_pt = mesh_pt()->element_pt(e);
6731 
6732  // Ignore halo elements
6733  if (!elem_pt->is_halo())
6734  {
6735  // Find number of degrees of freedom in the element
6736  const unsigned nvar = assembly_handler_pt->ndof(elem_pt);
6737 
6738  // Resize the storage for elemental jacobian and residuals
6739  for (unsigned v = 0; v < n_vector; v++)
6740  {
6741  el_residuals[v].resize(nvar);
6742  }
6743  for (unsigned m = 0; m < n_matrix; m++)
6744  {
6745  el_jacobian[m].resize(nvar);
6746  }
6747 
6748  // Now get the residuals and jacobian for the element
6750  elem_pt, el_residuals, el_jacobian);
6751 
6752  //---------------Insert the values into the vectors--------------
6753 
6754  // Loop over the first index of local variables
6755  for (unsigned i = 0; i < nvar; i++)
6756  {
6757  // Get the local equation number
6758  unsigned global_eqn_number =
6759  assembly_handler_pt->eqn_number(elem_pt, i);
6760 
6761  // determine the element number in my set of eqns using the
6762  // bisection method
6763  int left = 0;
6764  int right = my_n_eqn - 1;
6765  int eqn_number = right / 2;
6766  while (my_eqns[eqn_number] != global_eqn_number)
6767  {
6768  if (left == right)
6769  {
6770  // Check that the residuals associated with the
6771  // eqn number that can't be found are all zero
6772  bool broken = false;
6773  for (unsigned v = 0; v < n_vector; v++)
6774  {
6775  if (el_residuals[v][i] != 0.0)
6776  {
6777  broken = true;
6778  break;
6779  }
6780  }
6781 
6782  // Now loop over the other index to check the entries
6783  // in the appropriate row of the Jacobians are zero too
6784  for (unsigned j = 0; j < nvar; j++)
6785  {
6786  // Get the number of the unknown
6787  // unsigned unknown =
6788  // assembly_handler_pt->eqn_number(elem_pt,j);
6789 
6790  // Loop over the matrices
6791  // If it's compressed row storage, then our vector of maps
6792  // is indexed by row (equation number)
6793  for (unsigned m = 0; m < n_matrix; m++)
6794  {
6795  // Get the value of the matrix at this point
6796  double value = el_jacobian[m](i, j);
6797  if (value != 0.0)
6798  {
6799  broken = true;
6800  break;
6801  }
6802  if (broken) break;
6803  }
6804  }
6805 
6806  if (broken)
6807  {
6808  std::ostringstream error_stream;
6809  error_stream
6810  << "Internal Error: " << std::endl
6811  << "Could not find global equation number "
6812  << global_eqn_number
6813  << " in my_eqns vector of equation numbers but\n"
6814  << "at least one entry in the residual vector is nonzero.";
6815  throw OomphLibError(error_stream.str(),
6816  OOMPH_CURRENT_FUNCTION,
6817  OOMPH_EXCEPTION_LOCATION);
6818  }
6819  else
6820  {
6821  break;
6822  }
6823  }
6824  if (my_eqns[eqn_number] > global_eqn_number)
6825  {
6826  right = std::max(eqn_number - 1, left);
6827  }
6828  else
6829  {
6830  left = std::min(eqn_number + 1, right);
6831  }
6832  eqn_number = (right + left) / 2;
6833  }
6834 
6835  // Add the contribution to the residuals
6836  for (unsigned v = 0; v < n_vector; v++)
6837  {
6838  // Fill in each residuals vector
6839  residuals_data[v][eqn_number] += el_residuals[v][i];
6840  }
6841 
6842  // Now loop over the other index
6843  for (unsigned j = 0; j < nvar; j++)
6844  {
6845  // Get the number of the unknown
6846  unsigned unknown = assembly_handler_pt->eqn_number(elem_pt, j);
6847 
6848  // Loop over the matrices
6849  // If it's compressed row storage, then our vector of maps
6850  // is indexed by row (equation number)
6851  for (unsigned m = 0; m < n_matrix; m++)
6852  {
6853  // Get the value of the matrix at this point
6854  double value = el_jacobian[m](i, j);
6855  // Only bother to add to the vector if it's non-zero
6856  if (std::fabs(value) > Numerical_zero_for_sparse_assembly)
6857  {
6858  // number of entrys in this row
6859  const unsigned size = ncoef[m][eqn_number];
6860 
6861  // if no data has been allocated for this row then allocate
6862  if (size == 0)
6863  {
6864  // do we have previous allocation data
6866  [m][eqn_number] != 0)
6867  {
6868  matrix_col_indices[m][eqn_number] = new unsigned
6870  [m][eqn_number]];
6871 
6872  matrix_values[m][eqn_number] = new double
6874  [m][eqn_number]];
6875  }
6876  else
6877  {
6878  matrix_col_indices[m][eqn_number] = new unsigned
6880 
6881  matrix_values[m][eqn_number] = new double
6883 
6885  [m][eqn_number] =
6887  }
6888  }
6889 
6890  // next add the data
6891  for (unsigned k = 0; k <= size; k++)
6892  {
6893  if (k == size)
6894  {
6895  // do we need to allocate more storage
6897  [m][eqn_number] == ncoef[m][eqn_number])
6898  {
6899  unsigned new_allocation =
6900  ncoef[m][eqn_number] +
6902  double* new_values = new double[new_allocation];
6903  unsigned* new_indices = new unsigned[new_allocation];
6904  for (unsigned c = 0; c < ncoef[m][eqn_number]; c++)
6905  {
6906  new_values[c] = matrix_values[m][eqn_number][c];
6907  new_indices[c] = matrix_col_indices[m][eqn_number][c];
6908  }
6909  delete[] matrix_values[m][eqn_number];
6910  delete[] matrix_col_indices[m][eqn_number];
6911 
6912  matrix_values[m][eqn_number] = new_values;
6913  matrix_col_indices[m][eqn_number] = new_indices;
6914 
6916  [m][eqn_number] = new_allocation;
6917  }
6918  // and now add the data
6919  unsigned entry = ncoef[m][eqn_number];
6920  ncoef[m][eqn_number]++;
6921  matrix_col_indices[m][eqn_number][entry] = unknown;
6922  matrix_values[m][eqn_number][entry] = value;
6923  break;
6924  }
6925  else if (matrix_col_indices[m][eqn_number][k] == unknown)
6926  {
6927  matrix_values[m][eqn_number][k] += value;
6928  break;
6929  }
6930  }
6931  } // numerical zero check
6932  } // End of loop over matrices
6933  }
6934  }
6935  } // endif halo element
6936 
6937  // Time it?
6938  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
6939  {
6941  TimingHelpers::timer() - t_assemble_start;
6942  }
6943  } // End of loop over the elements
6944  } // End of vector assembly
6945 
6946 
6947  // Doc?
6948  double t_end = 0.0;
6949  double t_local = 0.0;
6950  double t_max = 0.0;
6951  double t_min = 0.0;
6952  double t_sum = 0.0;
6954  {
6955  t_end = TimingHelpers::timer();
6956  t_local = t_end - t_start;
6957  t_max = 0.0;
6958  t_min = 0.0;
6959  t_sum = 0.0;
6960  MPI_Allreduce(&t_local,
6961  &t_max,
6962  1,
6963  MPI_DOUBLE,
6964  MPI_MAX,
6965  this->communicator_pt()->mpi_comm());
6966  MPI_Allreduce(&t_local,
6967  &t_min,
6968  1,
6969  MPI_DOUBLE,
6970  MPI_MIN,
6971  this->communicator_pt()->mpi_comm());
6972  MPI_Allreduce(&t_local,
6973  &t_sum,
6974  1,
6975  MPI_DOUBLE,
6976  MPI_SUM,
6977  this->communicator_pt()->mpi_comm());
6978  double imbalance = (t_max - t_min) / (t_sum / double(nproc)) * 100.0;
6979 
6980  if (doing_residuals)
6981  {
6982  oomph_info << "\nCPU for residual computation (loc/max/min/imbal): ";
6983  }
6984  else
6985  {
6986  oomph_info << "\nCPU for Jacobian computation (loc/max/min/imbal): ";
6987  }
6988  oomph_info << t_local << " " << t_max << " " << t_min << " " << imbalance
6989  << "%\n";
6990 
6991  t_start = TimingHelpers::timer();
6992  }
6993 
6994 
6995  // Adjust number of coefficients in each row
6996  for (unsigned m = 0; m < n_matrix; m++)
6997  {
6998  unsigned max = 0;
6999  unsigned min = INT_MAX;
7000  unsigned sum = 0;
7001  unsigned sum_total = 0;
7002  for (unsigned e = 0; e < my_n_eqn; e++)
7003  {
7004  sum += ncoef[m][e];
7006  if (ncoef[m][e] > max) max = ncoef[m][e];
7007  if (ncoef[m][e] < min) min = ncoef[m][e];
7008 
7009  // Now shrink the storage to what we actually need
7010  unsigned new_allocation = ncoef[m][e];
7011  double* new_values = new double[new_allocation];
7012  unsigned* new_indices = new unsigned[new_allocation];
7013  for (unsigned c = 0; c < ncoef[m][e]; c++)
7014  {
7015  new_values[c] = matrix_values[m][e][c];
7016  new_indices[c] = matrix_col_indices[m][e][c];
7017  }
7018  delete[] matrix_values[m][e];
7019  delete[] matrix_col_indices[m][e];
7020 
7021  matrix_values[m][e] = new_values;
7022  matrix_col_indices[m][e] = new_indices;
7023  }
7024  }
7025 
7026 
7027  // Postprocess timing information and re-allocate distribution of
7028  // elements during subsequent assemblies.
7029  if ((!doing_residuals) && (!Problem_has_been_distributed) &&
7031  {
7033  }
7034 
7035  // We have determined load balancing for current setup.
7036  // This can remain the same until assign_eqn_numbers() is called
7037  // again -- the flag is re-set to true there.
7038  if ((!doing_residuals) && Must_recompute_load_balance_for_assembly)
7039  {
7041  }
7042 
7043 
7044  // next we compute the number of equations and number of non-zeros to be
7045  // sent to each processor, and send/recv that information
7046  // =====================================================================
7047 
7048  // determine the number of eqns to be sent to each processor
7049  Vector<unsigned> n_eqn_for_proc(nproc, 0);
7050  Vector<unsigned> first_eqn_element_for_proc(nproc, 0);
7051  // If no equations are assembled then we don't need to do any of this
7052  if (my_n_eqn > 0)
7053  {
7054  unsigned current_p = target_dist_pt->rank_of_global_row(my_eqns[0]);
7055  first_eqn_element_for_proc[current_p] = 0;
7056  n_eqn_for_proc[current_p] = 1;
7057  for (unsigned i = 1; i < my_n_eqn; i++)
7058  {
7059  unsigned next_p = target_dist_pt->rank_of_global_row(my_eqns[i]);
7060  if (next_p != current_p)
7061  {
7062  current_p = next_p;
7063  first_eqn_element_for_proc[current_p] = i;
7064  }
7065  n_eqn_for_proc[current_p]++;
7066  }
7067  }
7068 
7069  // determine the number of non-zeros to be sent to each processor for each
7070  // matrix (if n_eqn_for_proc[p]=0, then nothing will be assembled)
7071  DenseMatrix<unsigned> nnz_for_proc(nproc, n_matrix, 0);
7072  for (unsigned p = 0; p < nproc; p++)
7073  {
7074  int first_eqn_element = first_eqn_element_for_proc[p];
7075  int last_eqn_element = (int)(first_eqn_element + n_eqn_for_proc[p]) - 1;
7076  for (unsigned m = 0; m < n_matrix; m++)
7077  {
7078  for (int i = first_eqn_element; i <= last_eqn_element; i++)
7079  {
7080  nnz_for_proc(p, m) += ncoef[m][i];
7081  }
7082  }
7083  }
7084 
7085  // next post the sends and recvs to the corresponding processors
7086  Vector<unsigned*> temp_send_storage(nproc);
7087  Vector<unsigned*> temp_recv_storage(nproc);
7088  Vector<MPI_Request> send_nnz_reqs;
7089  Vector<MPI_Request> recv_nnz_reqs;
7090  for (unsigned p = 0; p < nproc; p++)
7091  {
7092  if (p != my_rank)
7093  {
7094  temp_send_storage[p] = new unsigned[n_matrix + 1];
7095  temp_send_storage[p][0] = n_eqn_for_proc[p];
7096  for (unsigned m = 0; m < n_matrix; m++)
7097  {
7098  temp_send_storage[p][m + 1] = nnz_for_proc(p, m);
7099  }
7100  MPI_Request sreq;
7101  MPI_Isend(temp_send_storage[p],
7102  n_matrix + 1,
7103  MPI_UNSIGNED,
7104  p,
7105  0,
7106  Communicator_pt->mpi_comm(),
7107  &sreq);
7108  send_nnz_reqs.push_back(sreq);
7109  temp_recv_storage[p] = new unsigned[n_matrix + 1];
7110  MPI_Request rreq;
7111  MPI_Irecv(temp_recv_storage[p],
7112  n_matrix + 1,
7113  MPI_UNSIGNED,
7114  p,
7115  0,
7116  Communicator_pt->mpi_comm(),
7117  &rreq);
7118  recv_nnz_reqs.push_back(rreq);
7119  }
7120  }
7121 
7122  // assemble the data to be sent to each processor
7123  // ==============================================
7124 
7125  // storage
7126  Vector<unsigned*> eqns_for_proc(nproc);
7127  DenseMatrix<double*> residuals_for_proc(nproc, n_vector);
7128  DenseMatrix<unsigned*> row_start_for_proc(nproc, n_matrix);
7129  DenseMatrix<unsigned*> column_indices_for_proc(nproc, n_matrix);
7130  DenseMatrix<double*> values_for_proc(nproc, n_matrix);
7131 
7132  // equation numbers
7133  for (unsigned p = 0; p < nproc; p++)
7134  {
7135  unsigned n_eqns_p = n_eqn_for_proc[p];
7136  if (n_eqns_p > 0)
7137  {
7138  unsigned first_eqn_element = first_eqn_element_for_proc[p];
7139  unsigned first_row = target_dist_pt->first_row(p);
7140  eqns_for_proc[p] = new unsigned[n_eqns_p];
7141  for (unsigned i = 0; i < n_eqns_p; i++)
7142  {
7143  eqns_for_proc[p][i] = my_eqns[i + first_eqn_element] - first_row;
7144  }
7145  }
7146  }
7147 
7148  // residuals for p
7149  for (unsigned v = 0; v < n_vector; v++)
7150  {
7151  for (unsigned p = 0; p < nproc; p++)
7152  {
7153  unsigned n_eqns_p = n_eqn_for_proc[p];
7154  if (n_eqns_p > 0)
7155  {
7156  unsigned first_eqn_element = first_eqn_element_for_proc[p];
7157  residuals_for_proc(p, v) = new double[n_eqns_p];
7158  for (unsigned i = 0; i < n_eqns_p; i++)
7159  {
7160  residuals_for_proc(p, v)[i] =
7161  residuals_data[v][first_eqn_element + i];
7162  }
7163  }
7164  }
7165  delete[] residuals_data[v];
7166  }
7167 
7168  // matrices for p
7169  for (unsigned m = 0; m < n_matrix; m++)
7170  {
7171  for (unsigned p = 0; p < nproc; p++)
7172  {
7173  unsigned n_eqns_p = n_eqn_for_proc[p];
7174  if (n_eqns_p > 0)
7175  {
7176  unsigned first_eqn_element = first_eqn_element_for_proc[p];
7177  row_start_for_proc(p, m) = new unsigned[n_eqns_p + 1];
7178  column_indices_for_proc(p, m) = new unsigned[nnz_for_proc(p, m)];
7179  values_for_proc(p, m) = new double[nnz_for_proc(p, m)];
7180  unsigned entry = 0;
7181  for (unsigned i = 0; i < n_eqns_p; i++)
7182  {
7183  row_start_for_proc(p, m)[i] = entry;
7184  unsigned n_coef_in_row = ncoef[m][first_eqn_element + i];
7185  for (unsigned j = 0; j < n_coef_in_row; j++)
7186  {
7187  column_indices_for_proc(p, m)[entry] =
7188  matrix_col_indices[m][i + first_eqn_element][j];
7189  values_for_proc(p, m)[entry] =
7190  matrix_values[m][i + first_eqn_element][j];
7191  entry++;
7192  }
7193  }
7194  row_start_for_proc(p, m)[n_eqns_p] = entry;
7195  }
7196  }
7197  for (unsigned i = 0; i < my_n_eqn; i++)
7198  {
7199  delete[] matrix_col_indices[m][i];
7200  delete[] matrix_values[m][i];
7201  }
7202  delete[] matrix_col_indices[m];
7203  delete[] matrix_values[m];
7204  }
7205 
7206  // need to wait for the recv nnzs to complete
7207  // before we can allocate storage for the matrix recvs
7208  // ===================================================
7209 
7210  // recv and copy the datafrom the recv storage to
7211  // + nnz_from_proc
7212  // + n_eqn_from_proc
7213  Vector<MPI_Status> recv_nnz_stat(nproc - 1);
7214  MPI_Waitall(nproc - 1, &recv_nnz_reqs[0], &recv_nnz_stat[0]);
7215  Vector<unsigned> n_eqn_from_proc(nproc);
7216  DenseMatrix<unsigned> nnz_from_proc(nproc, n_matrix);
7217  for (unsigned p = 0; p < nproc; p++)
7218  {
7219  if (p != my_rank)
7220  {
7221  n_eqn_from_proc[p] = temp_recv_storage[p][0];
7222  for (unsigned m = 0; m < n_matrix; m++)
7223  {
7224  nnz_from_proc(p, m) = temp_recv_storage[p][m + 1];
7225  }
7226  delete[] temp_recv_storage[p];
7227  }
7228  else
7229  {
7230  n_eqn_from_proc[p] = n_eqn_for_proc[p];
7231  for (unsigned m = 0; m < n_matrix; m++)
7232  {
7233  nnz_from_proc(p, m) = nnz_for_proc(p, m);
7234  }
7235  }
7236  }
7237  recv_nnz_stat.clear();
7238  recv_nnz_reqs.clear();
7239 
7240  // allocate the storage for the data to be recv and post the sends recvs
7241  // =====================================================================
7242 
7243  // storage
7244  Vector<unsigned*> eqns_from_proc(nproc);
7245  DenseMatrix<double*> residuals_from_proc(nproc, n_vector);
7246  DenseMatrix<unsigned*> row_start_from_proc(nproc, n_matrix);
7247  DenseMatrix<unsigned*> column_indices_from_proc(nproc, n_matrix);
7248  DenseMatrix<double*> values_from_proc(nproc, n_matrix);
7249 
7250  // allocate and post sends and recvs
7251  double base;
7252  MPI_Aint communication_base;
7253  MPI_Get_address(&base, &communication_base);
7254  unsigned n_comm_types = 1 + 1 * n_vector + 3 * n_matrix;
7255  Vector<MPI_Request> recv_reqs;
7256  Vector<MPI_Request> send_reqs;
7257  for (unsigned p = 0; p < nproc; p++)
7258  {
7259  if (p != my_rank)
7260  {
7261  // allocate
7262  if (n_eqn_from_proc[p] > 0)
7263  {
7264  eqns_from_proc[p] = new unsigned[n_eqn_from_proc[p]];
7265  for (unsigned v = 0; v < n_vector; v++)
7266  {
7267  residuals_from_proc(p, v) = new double[n_eqn_from_proc[p]];
7268  }
7269  for (unsigned m = 0; m < n_matrix; m++)
7270  {
7271  row_start_from_proc(p, m) = new unsigned[n_eqn_from_proc[p] + 1];
7272  column_indices_from_proc(p, m) = new unsigned[nnz_from_proc(p, m)];
7273  values_from_proc(p, m) = new double[nnz_from_proc(p, m)];
7274  }
7275  }
7276 
7277  // recv
7278  if (n_eqn_from_proc[p] > 0)
7279  {
7280  MPI_Datatype types[n_comm_types];
7281  MPI_Aint offsets[n_comm_types];
7282  int count[n_comm_types];
7283  int pt = 0;
7284 
7285  // equations
7286  count[pt] = 1;
7287  MPI_Get_address(eqns_from_proc[p], &offsets[pt]);
7288  offsets[pt] -= communication_base;
7289  MPI_Type_contiguous(n_eqn_from_proc[p], MPI_UNSIGNED, &types[pt]);
7290  MPI_Type_commit(&types[pt]);
7291  pt++;
7292 
7293  // vectors
7294  for (unsigned v = 0; v < n_vector; v++)
7295  {
7296  count[pt] = 1;
7297  MPI_Get_address(residuals_from_proc(p, v), &offsets[pt]);
7298  offsets[pt] -= communication_base;
7299  MPI_Type_contiguous(n_eqn_from_proc[p], MPI_DOUBLE, &types[pt]);
7300  MPI_Type_commit(&types[pt]);
7301  pt++;
7302  }
7303 
7304  // matrices
7305  for (unsigned m = 0; m < n_matrix; m++)
7306  {
7307  // row start
7308  count[pt] = 1;
7309  MPI_Get_address(row_start_from_proc(p, m), &offsets[pt]);
7310  offsets[pt] -= communication_base;
7311  MPI_Type_contiguous(
7312  n_eqn_from_proc[p] + 1, MPI_UNSIGNED, &types[pt]);
7313  MPI_Type_commit(&types[pt]);
7314  pt++;
7315 
7316 
7317  // column indices
7318  count[pt] = 1;
7319  MPI_Get_address(column_indices_from_proc(p, m), &offsets[pt]);
7320  offsets[pt] -= communication_base;
7321  MPI_Type_contiguous(nnz_from_proc(p, m), MPI_UNSIGNED, &types[pt]);
7322  MPI_Type_commit(&types[pt]);
7323  pt++;
7324 
7325  // values
7326  count[pt] = 1;
7327  MPI_Get_address(values_from_proc(p, m), &offsets[pt]);
7328  offsets[pt] -= communication_base;
7329  MPI_Type_contiguous(nnz_from_proc(p, m), MPI_DOUBLE, &types[pt]);
7330  MPI_Type_commit(&types[pt]);
7331  pt++;
7332  }
7333 
7334  // build the combined type
7335  MPI_Datatype recv_type;
7336  MPI_Type_create_struct(
7337  n_comm_types, count, offsets, types, &recv_type);
7338  MPI_Type_commit(&recv_type);
7339  for (unsigned t = 0; t < n_comm_types; t++)
7340  {
7341  MPI_Type_free(&types[t]);
7342  }
7343  MPI_Request req;
7344  MPI_Irecv(
7345  &base, 1, recv_type, p, 1, Communicator_pt->mpi_comm(), &req);
7346  MPI_Type_free(&recv_type);
7347  recv_reqs.push_back(req);
7348  }
7349 
7350  // send
7351  if (n_eqn_for_proc[p] > 0)
7352  {
7353  MPI_Datatype types[n_comm_types];
7354  MPI_Aint offsets[n_comm_types];
7355  int count[n_comm_types];
7356  int pt = 0;
7357 
7358  // equations
7359  count[pt] = 1;
7360  MPI_Get_address(eqns_for_proc[p], &offsets[pt]);
7361  offsets[pt] -= communication_base;
7362  MPI_Type_contiguous(n_eqn_for_proc[p], MPI_UNSIGNED, &types[pt]);
7363  MPI_Type_commit(&types[pt]);
7364  pt++;
7365 
7366  // vectors
7367  for (unsigned v = 0; v < n_vector; v++)
7368  {
7369  count[pt] = 1;
7370  MPI_Get_address(residuals_for_proc(p, v), &offsets[pt]);
7371  offsets[pt] -= communication_base;
7372  MPI_Type_contiguous(n_eqn_for_proc[p], MPI_DOUBLE, &types[pt]);
7373  MPI_Type_commit(&types[pt]);
7374  pt++;
7375  }
7376 
7377  // matrices
7378  for (unsigned m = 0; m < n_matrix; m++)
7379  {
7380  // row start
7381  count[pt] = 1;
7382  MPI_Get_address(row_start_for_proc(p, m), &offsets[pt]);
7383  offsets[pt] -= communication_base;
7384  MPI_Type_contiguous(
7385  n_eqn_for_proc[p] + 1, MPI_UNSIGNED, &types[pt]);
7386  MPI_Type_commit(&types[pt]);
7387  pt++;
7388 
7389 
7390  // column indices
7391  count[pt] = 1;
7392  MPI_Get_address(column_indices_for_proc(p, m), &offsets[pt]);
7393  offsets[pt] -= communication_base;
7394  MPI_Type_contiguous(nnz_for_proc(p, m), MPI_UNSIGNED, &types[pt]);
7395  MPI_Type_commit(&types[pt]);
7396  pt++;
7397 
7398  // values
7399  count[pt] = 1;
7400  MPI_Get_address(values_for_proc(p, m), &offsets[pt]);
7401  offsets[pt] -= communication_base;
7402  MPI_Type_contiguous(nnz_for_proc(p, m), MPI_DOUBLE, &types[pt]);
7403  MPI_Type_commit(&types[pt]);
7404  pt++;
7405  }
7406 
7407  // build the combined type
7408  MPI_Datatype send_type;
7409  MPI_Type_create_struct(
7410  n_comm_types, count, offsets, types, &send_type);
7411  MPI_Type_commit(&send_type);
7412  for (unsigned t = 0; t < n_comm_types; t++)
7413  {
7414  MPI_Type_free(&types[t]);
7415  }
7416  MPI_Request req;
7417  MPI_Isend(
7418  &base, 1, send_type, p, 1, Communicator_pt->mpi_comm(), &req);
7419  MPI_Type_free(&send_type);
7420  send_reqs.push_back(req);
7421  }
7422  }
7423  // otherwise send to self
7424  else
7425  {
7426  eqns_from_proc[p] = eqns_for_proc[p];
7427  for (unsigned v = 0; v < n_vector; v++)
7428  {
7429  residuals_from_proc(p, v) = residuals_for_proc(p, v);
7430  }
7431  for (unsigned m = 0; m < n_matrix; m++)
7432  {
7433  row_start_from_proc(p, m) = row_start_for_proc(p, m);
7434  column_indices_from_proc(p, m) = column_indices_for_proc(p, m);
7435  values_from_proc(p, m) = values_for_proc(p, m);
7436  }
7437  }
7438  }
7439 
7440  // wait for the recvs to complete
7441  unsigned n_recv_req = recv_reqs.size();
7442  if (n_recv_req > 0)
7443  {
7444  Vector<MPI_Status> recv_stat(n_recv_req);
7445  MPI_Waitall(n_recv_req, &recv_reqs[0], &recv_stat[0]);
7446  }
7447 
7448  // ==============================================
7449  unsigned target_nrow_local = target_dist_pt->nrow_local();
7450 
7451  // loop over the matrices
7452  for (unsigned m = 0; m < n_matrix; m++)
7453  {
7454  // allocate row_start
7455  row_start[m] = new int[target_nrow_local + 1];
7456  row_start[m][0] = 0;
7457 
7458  // initially allocate storage based on the maximum number of non-zeros
7459  // from any one processor
7460  unsigned nnz_allocation = Parallel_sparse_assemble_previous_allocation;
7461  for (unsigned p = 0; p < nproc; p++)
7462  {
7463  nnz_allocation = std::max(nnz_allocation, nnz_from_proc(p, m));
7464  }
7465  Vector<double*> values_chunk(1);
7466  values_chunk[0] = new double[nnz_allocation];
7467  Vector<int*> column_indices_chunk(1);
7468  column_indices_chunk[0] = new int[nnz_allocation];
7469  Vector<unsigned> ncoef_in_chunk(1, 0);
7470  Vector<unsigned> size_of_chunk(1, 0);
7471  size_of_chunk[0] = nnz_allocation;
7472  unsigned current_chunk = 0;
7473 
7474  // for each row on this processor
7475  for (unsigned i = 0; i < target_nrow_local; i++)
7476  {
7477  row_start[m][i] = 0;
7478 
7479  // determine the processors that this row is on
7480  Vector<int> row_on_proc(nproc, -1);
7481  for (unsigned p = 0; p < nproc; p++)
7482  {
7483  if (n_eqn_from_proc[p] == 0)
7484  {
7485  row_on_proc[p] = -1;
7486  }
7487  else
7488  {
7489  int left = 0;
7490  int right = n_eqn_from_proc[p] - 1;
7491  int midpoint = right / 2;
7492  bool complete = false;
7493  while (!complete)
7494  {
7495  midpoint = (right + left) / 2;
7496  if (midpoint > right)
7497  {
7498  midpoint = right;
7499  }
7500  if (midpoint < left)
7501  {
7502  midpoint = left;
7503  }
7504  if (left == right)
7505  {
7506  if (eqns_from_proc[p][midpoint] == i)
7507  {
7508  midpoint = left;
7509  }
7510  else
7511  {
7512  midpoint = -1;
7513  }
7514  complete = true;
7515  }
7516  else if (eqns_from_proc[p][midpoint] == i)
7517  {
7518  complete = true;
7519  }
7520  else if (eqns_from_proc[p][midpoint] > i)
7521  {
7522  right = std::max(midpoint - 1, left);
7523  }
7524  else
7525  {
7526  left = std::min(midpoint + 1, right);
7527  }
7528  }
7529  row_on_proc[p] = midpoint;
7530  }
7531  }
7532 
7533  // for each processor build this row of the matrix
7534  unsigned check_first = ncoef_in_chunk[current_chunk];
7535  unsigned check_last = check_first;
7536  for (unsigned p = 0; p < nproc; p++)
7537  {
7538  if (row_on_proc[p] != -1)
7539  {
7540  int row = row_on_proc[p];
7541  unsigned first = row_start_from_proc(p, m)[row];
7542  unsigned last = row_start_from_proc(p, m)[row + 1];
7543  for (unsigned l = first; l < last; l++)
7544  {
7545  bool done = false;
7546  for (unsigned j = check_first; j <= check_last && !done; j++)
7547  {
7548  if (j == check_last)
7549  {
7550  // is this temp array full, do we need to allocate
7551  // a new temp array
7552  if (ncoef_in_chunk[current_chunk] ==
7553  size_of_chunk[current_chunk])
7554  {
7555  // number of chunks allocated
7556  unsigned n_chunk = values_chunk.size();
7557 
7558  // determine the number of non-zeros added so far
7559  // (excluding the current row)
7560  unsigned nnz_so_far = 0;
7561  for (unsigned c = 0; c < n_chunk; c++)
7562  {
7563  nnz_so_far += ncoef_in_chunk[c];
7564  }
7565  nnz_so_far -= row_start[m][i];
7566 
7567  // average number of non-zeros per row
7568  unsigned avg_nnz = nnz_so_far / (i + 1);
7569 
7570  // number of rows left +1
7571  unsigned nrows_left = target_nrow_local - i;
7572 
7573  // allocation for next chunk
7574  unsigned next_chunk_size =
7575  avg_nnz * nrows_left + row_start[m][i];
7576 
7577  // allocate storage in next chunk
7578  current_chunk++;
7579  n_chunk++;
7580  values_chunk.resize(n_chunk);
7581  values_chunk[current_chunk] = new double[next_chunk_size];
7582  column_indices_chunk.resize(n_chunk);
7583  column_indices_chunk[current_chunk] =
7584  new int[next_chunk_size];
7585  size_of_chunk.resize(n_chunk);
7586  size_of_chunk[current_chunk] = next_chunk_size;
7587  ncoef_in_chunk.resize(n_chunk);
7588 
7589  // copy current row from previous chunk to new chunk
7590  for (unsigned k = check_first; k < check_last; k++)
7591  {
7592  values_chunk[current_chunk][k - check_first] =
7593  values_chunk[current_chunk - 1][k];
7594  column_indices_chunk[current_chunk][k - check_first] =
7595  column_indices_chunk[current_chunk - 1][k];
7596  }
7597  ncoef_in_chunk[current_chunk - 1] -= row_start[m][i];
7598  ncoef_in_chunk[current_chunk] = row_start[m][i];
7599 
7600  // update first_check and last_check
7601  check_first = 0;
7602  check_last = row_start[m][i];
7603  j = check_last;
7604  }
7605 
7606  // add the coefficient
7607  values_chunk[current_chunk][j] = values_from_proc(p, m)[l];
7608  column_indices_chunk[current_chunk][j] =
7609  column_indices_from_proc(p, m)[l];
7610  ncoef_in_chunk[current_chunk]++;
7611  row_start[m][i]++;
7612  check_last++;
7613  done = true;
7614  }
7615  else if (column_indices_chunk[current_chunk][j] ==
7616  (int)column_indices_from_proc(p, m)[l])
7617  {
7618  values_chunk[current_chunk][j] += values_from_proc(p, m)[l];
7619  done = true;
7620  }
7621  }
7622  }
7623  }
7624  }
7625  }
7626 
7627  // delete recv data for this matrix
7628  for (unsigned p = 0; p < nproc; p++)
7629  {
7630  if (n_eqn_from_proc[p] > 0)
7631  {
7632  delete[] row_start_from_proc(p, m);
7633  delete[] column_indices_from_proc(p, m);
7634  delete[] values_from_proc(p, m);
7635  }
7636  }
7637 
7638  // next we take the chunk base storage of the column indices and values
7639  // and copy into a single contiguous block of memory
7640  // ====================================================================
7641  unsigned n_chunk = values_chunk.size();
7642  nnz[m] = 0;
7643  for (unsigned c = 0; c < n_chunk; c++)
7644  {
7645  nnz[m] += ncoef_in_chunk[c];
7646  }
7648 
7649  // allocate
7650  values[m] = new double[nnz[m]];
7651  column_indices[m] = new int[nnz[m]];
7652 
7653  // copy
7654  unsigned pt = 0;
7655  for (unsigned c = 0; c < n_chunk; c++)
7656  {
7657  unsigned nc = ncoef_in_chunk[c];
7658  for (unsigned i = 0; i < nc; i++)
7659  {
7660  values[m][pt + i] = values_chunk[c][i];
7661  column_indices[m][pt + i] = column_indices_chunk[c][i];
7662  }
7663  pt += nc;
7664  delete[] values_chunk[c];
7665  delete[] column_indices_chunk[c];
7666  }
7667 
7668  // the row_start vector currently contains the number of coefs in each
7669  // row. Update
7670  // ===================================================================
7671  unsigned g = row_start[m][0];
7672  row_start[m][0] = 0;
7673  for (unsigned i = 1; i < target_nrow_local; i++)
7674  {
7675  unsigned h = g + row_start[m][i];
7676  row_start[m][i] = g;
7677  g = h;
7678  }
7679  row_start[m][target_nrow_local] = g;
7680  }
7681 
7682  // next accumulate the residuals
7683  for (unsigned v = 0; v < n_vector; v++)
7684  {
7685  residuals[v] = new double[target_nrow_local];
7686  for (unsigned i = 0; i < target_nrow_local; i++)
7687  {
7688  residuals[v][i] = 0;
7689  }
7690  for (unsigned p = 0; p < nproc; p++)
7691  {
7692  if (n_eqn_from_proc[p] > 0)
7693  {
7694  unsigned n_eqn_p = n_eqn_from_proc[p];
7695  for (unsigned i = 0; i < n_eqn_p; i++)
7696  {
7697  residuals[v][eqns_from_proc[p][i]] += residuals_from_proc(p, v)[i];
7698  }
7699  delete[] residuals_from_proc(p, v);
7700  }
7701  }
7702  }
7703 
7704  // delete list of eqns from proc
7705  for (unsigned p = 0; p < nproc; p++)
7706  {
7707  if (n_eqn_from_proc[p] > 0)
7708  {
7709  delete[] eqns_from_proc[p];
7710  }
7711  }
7712 
7713  // and wait for sends to complete
7714  Vector<MPI_Status> send_nnz_stat(nproc - 1);
7715  MPI_Waitall(nproc - 1, &send_nnz_reqs[0], &send_nnz_stat[0]);
7716  for (unsigned p = 0; p < nproc; p++)
7717  {
7718  if (p != my_rank)
7719  {
7720  delete[] temp_send_storage[p];
7721  }
7722  }
7723  send_nnz_stat.clear();
7724  send_nnz_reqs.clear();
7725 
7726  // wait for the matrix data sends to complete and delete the data
7727  unsigned n_send_reqs = send_reqs.size();
7728  if (n_send_reqs > 0)
7729  {
7730  Vector<MPI_Status> send_stat(n_send_reqs);
7731  MPI_Waitall(n_send_reqs, &send_reqs[0], &send_stat[0]);
7732  for (unsigned p = 0; p < nproc; p++)
7733  {
7734  if (p != my_rank)
7735  {
7736  if (n_eqn_for_proc[p])
7737  {
7738  delete[] eqns_for_proc[p];
7739  for (unsigned m = 0; m < n_matrix; m++)
7740  {
7741  delete[] row_start_for_proc(p, m);
7742  delete[] column_indices_for_proc(p, m);
7743  delete[] values_for_proc(p, m);
7744  }
7745  for (unsigned v = 0; v < n_vector; v++)
7746  {
7747  delete[] residuals_for_proc(p, v);
7748  }
7749  }
7750  }
7751  }
7752  }
7753 
7754  // Doc?
7756  {
7757  t_end = TimingHelpers::timer();
7758  t_local = t_end - t_start;
7759  t_max = 0.0;
7760  t_min = 0.0;
7761  t_sum = 0.0;
7762  MPI_Allreduce(&t_local,
7763  &t_max,
7764  1,
7765  MPI_DOUBLE,
7766  MPI_MAX,
7767  this->communicator_pt()->mpi_comm());
7768  MPI_Allreduce(&t_local,
7769  &t_min,
7770  1,
7771  MPI_DOUBLE,
7772  MPI_MIN,
7773  this->communicator_pt()->mpi_comm());
7774  MPI_Allreduce(&t_local,
7775  &t_sum,
7776  1,
7777  MPI_DOUBLE,
7778  MPI_SUM,
7779  this->communicator_pt()->mpi_comm());
7780  double imbalance = (t_max - t_min) / (t_sum / double(nproc)) * 100.0;
7781  if (doing_residuals)
7782  {
7783  oomph_info << "CPU for residual distribut. (loc/max/min/imbal): ";
7784  }
7785  else
7786  {
7787  oomph_info << "CPU for Jacobian distribut. (loc/max/min/imbal): ";
7788  }
7789  oomph_info << t_local << " " << t_max << " " << t_min << " " << imbalance
7790  << "%\n\n";
7791  }
7792  }
7793 
7794 #endif
7795 
7796 
7797  //================================================================
7798  /// Get the full Jacobian by finite differencing
7799  //================================================================
7801  DenseMatrix<double>& jacobian)
7802  {
7803 #ifdef OOMPH_HAS_MPI
7804 
7806  {
7807  OomphLibWarning("This is unlikely to work with a distributed problem",
7808  " Problem::get_fd_jacobian()",
7809  OOMPH_EXCEPTION_LOCATION);
7810  }
7811 #endif
7812 
7813 
7814  // Find number of dofs
7815  const unsigned long n_dof = ndof();
7816 
7817  // Advanced residuals
7818  DoubleVector residuals_pls;
7819 
7820  // Get reference residuals
7821  get_residuals(residuals);
7822 
7823  const double FD_step = 1.0e-8;
7824 
7825  // Make sure the Jacobian is the right size (since we don't care about
7826  // speed).
7827  jacobian.resize(n_dof, n_dof);
7828 
7829  // Loop over all dofs
7830  for (unsigned long jdof = 0; jdof < n_dof; jdof++)
7831  {
7832  double backup = *Dof_pt[jdof];
7833  *Dof_pt[jdof] += FD_step;
7834 
7835  // We're checking if the new values for Dof_pt[] actually
7836  // solve the entire problem --> update as if problem had
7837  // been solved
7841 
7842  // Get advanced residuals
7843  get_residuals(residuals_pls);
7844 
7845  for (unsigned long ieqn = 0; ieqn < n_dof; ieqn++)
7846  {
7847  jacobian(ieqn, jdof) =
7848  (residuals_pls[ieqn] - residuals[ieqn]) / FD_step;
7849  }
7850 
7851  *Dof_pt[jdof] = backup;
7852  }
7853 
7854  // Reset problem to state it was in
7858  }
7859 
7860  //======================================================================
7861  /// Get derivative of the residuals vector wrt a global parameter
7862  /// This is required in continuation problems
7863  //=======================================================================
7864  void Problem::get_derivative_wrt_global_parameter(double* const& parameter_pt,
7865  DoubleVector& result)
7866  {
7867  // If we are doing the calculation analytically then call the appropriate
7868  // handler and then calling get_residuals
7869  if (is_dparameter_calculated_analytically(parameter_pt))
7870  {
7871  // Locally cache pointer to assembly handler
7872  AssemblyHandler* const old_assembly_handler_pt = Assembly_handler_pt;
7873  // Create a new assembly handler that replaces get_residuals by
7874  // get_dresiduals_dparameter for each element
7876  new ParameterDerivativeHandler(old_assembly_handler_pt, parameter_pt);
7877  // Get the residuals, which will be dresiduals by dparameter
7878  this->get_residuals(result);
7879  // Delete the parameter derivative handler
7880  delete Assembly_handler_pt;
7881  // Reset the assembly handler to the original handler
7882  Assembly_handler_pt = old_assembly_handler_pt;
7883 
7884  /*AssemblyHandler* const assembly_handler_pt = Assembly_handler_pt;
7885  //Loop over all the elements
7886  unsigned long Element_pt_range = Mesh_pt->nelement();
7887  for(unsigned long e=0;e<Element_pt_range;e++)
7888  {
7889  //Get the pointer to the element
7890  GeneralisedElement* elem_pt = Mesh_pt->element_pt(e);
7891  //Find number of dofs in the element
7892  unsigned n_element_dofs = assembly_handler_pt->ndof(elem_pt);
7893  //Set up an array
7894  Vector<double> element_residuals(n_element_dofs);
7895  //Fill the array
7896  assembly_handler_pt->get_dresiduals_dparameter(elem_pt,parameter_pt,
7897  element_residuals);
7898  //Now loop over the dofs and assign values to global Vector
7899  for(unsigned l=0;l<n_element_dofs;l++)
7900  {
7901  result[assembly_handler_pt->eqn_number(elem_pt,l)]
7902  += element_residuals[l];
7903  }
7904  }*/
7905 
7906  // for(unsigned n=0;n<n_dof;n++)
7907  // {std::cout << "BLA " << n << " " << result[n] << "\n";}
7908  }
7909  // Otherwise use the finite difference default
7910  else
7911  {
7912  // Get the (global) residuals and store in the result vector
7913  get_residuals(result);
7914 
7915  // Storage for the new residuals
7916  DoubleVector newres;
7917 
7918  // Increase the global parameter
7919  const double FD_step = 1.0e-8;
7920 
7921  // Store the current value of the parameter
7922  double param_value = *parameter_pt;
7923 
7924  // Increase the parameter
7925  *parameter_pt += FD_step;
7926 
7927  // Do any possible updates
7929 
7930  // Get the new residuals
7931  get_residuals(newres);
7932 
7933  // Find the number of local rows
7934  //(I think it's a global vector, so that should be fine)
7935  const unsigned ndof_local = result.nrow_local();
7936 
7937  // Do the finite differencing in the local variables
7938  for (unsigned n = 0; n < ndof_local; ++n)
7939  {
7940  result[n] = (newres[n] - result[n]) / FD_step;
7941  }
7942 
7943  // Reset the value of the parameter
7944  *parameter_pt = param_value;
7945 
7946  // Do any possible updates
7948  }
7949  }
7950 
7951 
7952  //======================================================================
7953  /// Return the product of the global hessian (derivative of Jacobian
7954  /// matrix with respect to all variables) with
7955  /// an eigenvector, Y, and any number of other specified vectors C
7956  /// (d(J_{ij})/d u_{k}) Y_{j} C_{k}.
7957  /// This function is used in assembling and solving the augmented systems
7958  /// associated with bifurcation tracking.
7959  /// The default implementation is to use finite differences at the global
7960  /// level.
7961  //========================================================================
7963  DoubleVectorWithHaloEntries const& Y,
7966  {
7967  // How many vector products must we construct
7968  const unsigned n_vec = C.size();
7969 
7970  // currently only global (non-distributed) distributions are allowed
7971  // LinearAlgebraDistribution* dist_pt = new
7972  // LinearAlgebraDistribution(Communicator_pt,n_dof,false);
7973 
7974  // Cache the assembly hander
7976 
7977  // Rebuild the results vectors and initialise to zero
7978  // use the same distribution of the vector Y
7979  for (unsigned i = 0; i < n_vec; i++)
7980  {
7981  product[i].build(Y.distribution_pt(), 0.0);
7982  product[i].initialise(0.0);
7983  }
7984 
7985 // Setup the halo schemes for the result
7986 #ifdef OOMPH_HAS_MPI
7988  {
7989  for (unsigned i = 0; i < n_vec; i++)
7990  {
7991  product[i].build_halo_scheme(this->Halo_scheme_pt);
7992  }
7993  }
7994 #endif
7995 
7996  // If we are doing the calculation analytically then call the appropriate
7997  // handler
7998  // A better way to do this is probably to hook into the get_residuals
7999  // framework but with a different member function of the assembly
8000  // handler
8002  {
8003  // Loop over all the elements
8004  unsigned long Element_pt_range = Mesh_pt->nelement();
8005  for (unsigned long e = 0; e < Element_pt_range; e++)
8006  {
8007  // Get the pointer to the element
8008  GeneralisedElement* elem_pt = Mesh_pt->element_pt(e);
8009 // Do not loop over halo elements
8010 #ifdef OOMPH_HAS_MPI
8011  if (!elem_pt->is_halo())
8012  {
8013 #endif
8014  // Find number of dofs in the element
8015  unsigned n_var = assembly_handler_pt->ndof(elem_pt);
8016  // Set up a matrix for the input and output
8017  Vector<double> Y_local(n_var);
8018  DenseMatrix<double> C_local(n_vec, n_var);
8019  DenseMatrix<double> product_local(n_vec, n_var);
8020 
8021  // Translate the global input vectors into the local storage
8022  // Probably horribly inefficient, but otherwise things get really
8023  // messy at the elemental level
8024  for (unsigned l = 0; l < n_var; l++)
8025  {
8026  // Cache the global equation number
8027  const unsigned long eqn_number =
8028  assembly_handler_pt->eqn_number(elem_pt, l);
8029 
8030  Y_local[l] = Y.global_value(eqn_number);
8031  for (unsigned i = 0; i < n_vec; i++)
8032  {
8033  C_local(i, l) = C[i].global_value(eqn_number);
8034  }
8035  }
8036 
8037  // Fill the array
8039  elem_pt, Y_local, C_local, product_local);
8040 
8041  // Assign the local results to the global vector
8042  for (unsigned l = 0; l < n_var; l++)
8043  {
8044  const unsigned long eqn_number =
8045  assembly_handler_pt->eqn_number(elem_pt, l);
8046 
8047  for (unsigned i = 0; i < n_vec; i++)
8048  {
8049  product[i].global_value(eqn_number) += product_local(i, l);
8050  // std::cout << "BLA " << e << " " << i << " "
8051  // << l << " " << product_local(i,l) << "\n";
8052  }
8053  }
8054 #ifdef OOMPH_HAS_MPI
8055  }
8056 #endif
8057  }
8058  }
8059  // Otherwise calculate using finite differences by
8060  // perturbing the jacobian along a particular direction
8061  else
8062  {
8063  // Cache the finite difference step
8064  /// Alice: My bifurcation tracking converges better with this FD_step
8065  /// as 1.0e-5. The default value remains at 1.0e-8.
8067 
8068  // We can now construct our multipliers
8069  const unsigned n_dof_local = this->Dof_distribution_pt->nrow_local();
8070  // Prepare to scale
8071  double dof_length = 0.0;
8072  Vector<double> C_length(n_vec, 0.0);
8073 
8074  for (unsigned n = 0; n < n_dof_local; n++)
8075  {
8076  if (std::fabs(this->dof(n)) > dof_length)
8077  {
8078  dof_length = std::fabs(this->dof(n));
8079  }
8080  }
8081 
8082  // C is assumed to have the same distribution as the dofs
8083  for (unsigned i = 0; i < n_vec; i++)
8084  {
8085  for (unsigned n = 0; n < n_dof_local; n++)
8086  {
8087  if (std::fabs(C[i][n]) > C_length[i])
8088  {
8089  C_length[i] = std::fabs(C[i][n]);
8090  }
8091  }
8092  }
8093 
8094  // Now broadcast the information, if distributed
8095 #ifdef OOMPH_HAS_MPI
8097  {
8098  const unsigned n_length = n_vec + 1;
8099  double all_length[n_length];
8100  all_length[0] = dof_length;
8101  for (unsigned i = 0; i < n_vec; i++)
8102  {
8103  all_length[i + 1] = C_length[i];
8104  }
8105 
8106  // Do the MPI call
8107  double all_length_reduce[n_length];
8108  MPI_Allreduce(all_length,
8109  all_length_reduce,
8110  n_length,
8111  MPI_DOUBLE,
8112  MPI_MAX,
8113  this->communicator_pt()->mpi_comm());
8114 
8115  // Read out the information
8116  dof_length = all_length_reduce[0];
8117  for (unsigned i = 0; i < n_vec; i++)
8118  {
8119  C_length[i] = all_length_reduce[i + 1];
8120  }
8121  }
8122 #endif
8123 
8124  // Form the multipliers
8125  Vector<double> C_mult(n_vec, 0.0);
8126  for (unsigned i = 0; i < n_vec; i++)
8127  {
8128  C_mult[i] = dof_length / C_length[i];
8129  C_mult[i] += FD_step;
8130  C_mult[i] *= FD_step;
8131  }
8132 
8133 
8134  // Dummy vector to stand in the place of the residuals
8135  Vector<double> dummy_res;
8136 
8137  // Calculate the product of the jacobian matrices, etc by looping over the
8138  // elements
8139  const unsigned long n_element = this->mesh_pt()->nelement();
8140  for (unsigned long e = 0; e < n_element; e++)
8141  {
8142  GeneralisedElement* elem_pt = this->mesh_pt()->element_pt(e);
8143  // Ignore halo's of course
8144 #ifdef OOMPH_HAS_MPI
8145  if (!elem_pt->is_halo())
8146  {
8147 #endif
8148  // Loop over the ndofs in each element
8149  unsigned n_var = assembly_handler_pt->ndof(elem_pt);
8150  // Resize the dummy residuals vector
8151  dummy_res.resize(n_var);
8152  // Allocate storage for the unperturbed jacobian matrix
8153  DenseMatrix<double> jac(n_var);
8154  // Get unperturbed jacobian
8155  assembly_handler_pt->get_jacobian(elem_pt, dummy_res, jac);
8156 
8157  // Backup the dofs
8158  Vector<double> dof_bac(n_var);
8159  for (unsigned n = 0; n < n_var; n++)
8160  {
8161  unsigned eqn_number = assembly_handler_pt->eqn_number(elem_pt, n);
8162  dof_bac[n] = *this->global_dof_pt(eqn_number);
8163  }
8164 
8165  // Now loop over all vectors C
8166  for (unsigned i = 0; i < n_vec; i++)
8167  {
8168  // Perturb the dofs by the appropriate vector
8169  for (unsigned n = 0; n < n_var; n++)
8170  {
8171  unsigned eqn_number = assembly_handler_pt->eqn_number(elem_pt, n);
8172  // Perturb by vector C[i]
8173  *this->global_dof_pt(eqn_number) +=
8174  C_mult[i] * C[i].global_value(eqn_number);
8175  }
8177 
8178  // Allocate storage for the perturbed jacobian
8179  DenseMatrix<double> jac_C(n_var);
8180 
8181  // Now get the new jacobian
8182  assembly_handler_pt->get_jacobian(elem_pt, dummy_res, jac_C);
8183 
8184  // Reset the dofs
8185  for (unsigned n = 0; n < n_var; n++)
8186  {
8187  unsigned eqn_number = assembly_handler_pt->eqn_number(elem_pt, n);
8188  *this->global_dof_pt(eqn_number) = dof_bac[n];
8189  }
8191 
8192  // Now work out the products
8193  for (unsigned n = 0; n < n_var; n++)
8194  {
8195  unsigned eqn_number = assembly_handler_pt->eqn_number(elem_pt, n);
8196  double prod_c = 0.0;
8197  for (unsigned m = 0; m < n_var; m++)
8198  {
8199  unsigned unknown = assembly_handler_pt->eqn_number(elem_pt, m);
8200  prod_c += (jac_C(n, m) - jac(n, m)) * Y.global_value(unknown);
8201  }
8202  // std::cout << "FD " << e << " " << i << " "
8203  // << n << " " << prod_c/C_mult[i] << "\n";
8204  product[i].global_value(eqn_number) += prod_c / C_mult[i];
8205  }
8206  }
8207 #ifdef OOMPH_HAS_MPI
8208  }
8209 #endif
8210  } // End of loop over elements
8211  }
8212 
8213  // If we have a distributed problem then gather all
8214  // values
8215 #ifdef OOMPH_HAS_MPI
8217  {
8218  // Sum all values if distributed
8219  for (unsigned i = 0; i < n_vec; i++)
8220  {
8221  product[i].sum_all_halo_and_haloed_values();
8222  }
8223  }
8224 #endif
8225  }
8226 
8227 
8228  //================================================================
8229  /// Get derivative of an element in the problem wrt a global
8230  /// parameter, to be used in continuation problems
8231  //================================================================
8232  /*void Problem::get_derivative_wrt_global_parameter(
8233  double* const &parameter_pt,
8234  GeneralisedElement* const &elem_pt,
8235  Vector<double> &result)
8236  {
8237 
8238  #ifdef OOMPH_HAS_MPI
8239 
8240  if (Problem_has_been_distributed)
8241  {
8242  OomphLibWarning("This is unlikely to work with a distributed problem",
8243  "Problem::get_derivative_wrt_global_parameter()",
8244  OOMPH_EXCEPTION_LOCATION);
8245  }
8246  #endif
8247 
8248  //Locally cache pointer to assembly handler
8249  AssemblyHandler* const assembly_handler_pt = Assembly_handler_pt;
8250 
8251  //Should definitely give this a more global scope
8252  double FD_Jstep = 1.0e-8;
8253 
8254  //Find the number of variables in the element, e
8255  unsigned nvar = assembly_handler_pt->ndof(elem_pt);
8256  //Create storage for residuals
8257  Vector<double> residuals(nvar), newres(nvar);
8258 
8259  //Get the "original" residuals
8260  assembly_handler_pt->get_residuals(elem_pt,residuals);
8261 
8262  //Save the old value of the global parameter
8263  double old_var = *parameter_pt;
8264 
8265  //Increment the value
8266  *parameter_pt += FD_Jstep;
8267 
8268  //Now do any possible updates
8269  actions_after_change_in_global_parameter();
8270 
8271  //Get the "new" residuals
8272  assembly_handler_pt->get_residuals(elem_pt,newres);
8273 
8274  //Do the finite differences
8275  for(unsigned m=0;m<nvar;m++)
8276  {
8277  result[m] = (newres[m] - residuals[m])/FD_Jstep;
8278  }
8279 
8280  //Reset value of the global parameter
8281  *parameter_pt = old_var;
8282 
8283  //Now do any possible updates
8284  actions_after_change_in_global_parameter();
8285  }*/
8286 
8287  //==================================================================
8288  /// Solve the eigenproblem. Legacy version that returns real vectors which are
8289  /// related in some solver-specific way to the real and imaginary parts
8290  /// of the actual, usually complex eigenvalues.
8291  /// At least n_eval eigenvalues are computed.
8292  //==================================================================
8294  const unsigned& n_eval,
8295  Vector<std::complex<double>>& eigenvalue,
8296  Vector<DoubleVector>& eigenvector,
8297  const bool& make_timesteppers_steady)
8298  {
8299  // If the boolean flag is steady, then make all the timesteppers steady
8300  // before solving the eigenproblem. This will "switch off" the
8301  // time-derivative terms in the jacobian matrix
8302  if (make_timesteppers_steady)
8303  {
8304  // Find out how many timesteppers there are
8305  const unsigned n_time_steppers = ntime_stepper();
8306 
8307  // Vector of bools to store the is_steady status of the various
8308  // timesteppers when we came in here
8309  std::vector<bool> was_steady(n_time_steppers);
8310 
8311  // Loop over them all and make them (temporarily) static
8312  for (unsigned i = 0; i < n_time_steppers; i++)
8313  {
8314  was_steady[i] = time_stepper_pt(i)->is_steady();
8316  }
8317 
8318  const bool do_adjoint_problem = false;
8319  // Call the Eigenproblem for the eigensolver
8321  this, n_eval, eigenvalue, eigenvector, do_adjoint_problem);
8322 
8323  // Reset the is_steady status of all timesteppers that
8324  // weren't already steady when we came in here and reset their
8325  // weights
8326  for (unsigned i = 0; i < n_time_steppers; i++)
8327  {
8328  if (!was_steady[i])
8329  {
8331  }
8332  }
8333  }
8334  // Otherwise if we don't want to make the problem steady, just
8335  // assemble and solve the eigensystem
8336  else
8337  {
8338  const bool do_adjoint_problem = false;
8339  // Call the Eigenproblem for the eigensolver
8341  this, n_eval, eigenvalue, eigenvector, do_adjoint_problem);
8342  }
8343  }
8344 
8345  //==================================================================
8346  /// Solve the eigenproblem
8347  //==================================================================
8348  void Problem::solve_eigenproblem(const unsigned& n_eval,
8349  Vector<std::complex<double>>& alpha,
8350  Vector<double>& beta,
8351  Vector<DoubleVector>& eigenvector_real,
8352  Vector<DoubleVector>& eigenvector_imag,
8353  const bool& make_timesteppers_steady)
8354  {
8355  // If the boolean flag is steady, then make all the timesteppers steady
8356  // before solving the eigenproblem. This will "switch off" the
8357  // time-derivative terms in the jacobian matrix
8358  if (make_timesteppers_steady)
8359  {
8360  // Find out how many timesteppers there are
8361  const unsigned n_time_steppers = ntime_stepper();
8362 
8363  // Vector of bools to store the is_steady status of the various
8364  // timesteppers when we came in here
8365  std::vector<bool> was_steady(n_time_steppers);
8366 
8367  // Loop over them all and make them (temporarily) static
8368  for (unsigned i = 0; i < n_time_steppers; i++)
8369  {
8370  was_steady[i] = time_stepper_pt(i)->is_steady();
8372  }
8373 
8374  const bool do_adjoint_problem = false;
8375  // Call the Eigenproblem for the eigensolver
8377  n_eval,
8378  alpha,
8379  beta,
8380  eigenvector_real,
8381  eigenvector_imag,
8382  do_adjoint_problem);
8383 
8384  // Reset the is_steady status of all timesteppers that
8385  // weren't already steady when we came in here and reset their
8386  // weights
8387  for (unsigned i = 0; i < n_time_steppers; i++)
8388  {
8389  if (!was_steady[i])
8390  {
8392  }
8393  }
8394  }
8395  // Otherwise if we don't want to make the problem steady, just
8396  // assemble and solve the eigensystem
8397  else
8398  {
8399  const bool do_adjoint_problem = false;
8400  // Call the Eigenproblem for the eigensolver
8402  n_eval,
8403  alpha,
8404  beta,
8405  eigenvector_real,
8406  eigenvector_imag,
8407  do_adjoint_problem);
8408  }
8409  }
8410 
8411 
8412  //==================================================================
8413  /// Solve the eigenproblem
8414  //==================================================================
8415  void Problem::solve_eigenproblem(const unsigned& n_eval,
8416  Vector<std::complex<double>>& eigenvalue,
8417  Vector<DoubleVector>& eigenvector_real,
8418  Vector<DoubleVector>& eigenvector_imag,
8419  const bool& make_timesteppers_steady)
8420  {
8421  // If the boolean flag is steady, then make all the timesteppers steady
8422  // before solving the eigenproblem. This will "switch off" the
8423  // time-derivative terms in the jacobian matrix
8424  if (make_timesteppers_steady)
8425  {
8426  // Find out how many timesteppers there are
8427  const unsigned n_time_steppers = ntime_stepper();
8428 
8429  // Vector of bools to store the is_steady status of the various
8430  // timesteppers when we came in here
8431  std::vector<bool> was_steady(n_time_steppers);
8432 
8433  // Loop over them all and make them (temporarily) static
8434  for (unsigned i = 0; i < n_time_steppers; i++)
8435  {
8436  was_steady[i] = time_stepper_pt(i)->is_steady();
8438  }
8439 
8440  const bool do_adjoint_problem = false;
8441  // Call the Eigenproblem for the eigensolver
8443  n_eval,
8444  eigenvalue,
8445  eigenvector_real,
8446  eigenvector_imag,
8447  do_adjoint_problem);
8448 
8449  // Reset the is_steady status of all timesteppers that
8450  // weren't already steady when we came in here and reset their
8451  // weights
8452  for (unsigned i = 0; i < n_time_steppers; i++)
8453  {
8454  if (!was_steady[i])
8455  {
8457  }
8458  }
8459  }
8460  // Otherwise if we don't want to make the problem steady, just
8461  // assemble and solve the eigensystem
8462  else
8463  {
8464  const bool do_adjoint_problem = false;
8465  // Call the Eigenproblem for the eigensolver
8467  n_eval,
8468  eigenvalue,
8469  eigenvector_real,
8470  eigenvector_imag,
8471  do_adjoint_problem);
8472  }
8473  }
8474 
8475 
8476  //==================================================================
8477  /// Solve the adjoint eigenproblem
8478  //==================================================================
8480  const unsigned& n_eval,
8481  Vector<std::complex<double>>& eigenvalue,
8482  Vector<DoubleVector>& eigenvector,
8483  const bool& make_timesteppers_steady)
8484  {
8485  // If the boolean flag is steady, then make all the timesteppers steady
8486  // before solving the eigenproblem. This will "switch off" the
8487  // time-derivative terms in the jacobian matrix
8488  if (make_timesteppers_steady)
8489  {
8490  // Find out how many timesteppers there are
8491  const unsigned n_time_steppers = ntime_stepper();
8492 
8493  // Vector of bools to store the is_steady status of the various
8494  // timesteppers when we came in here
8495  std::vector<bool> was_steady(n_time_steppers);
8496 
8497  // Loop over them all and make them (temporarily) static
8498  for (unsigned i = 0; i < n_time_steppers; i++)
8499  {
8500  was_steady[i] = time_stepper_pt(i)->is_steady();
8502  }
8503 
8504  const bool do_adjoint_problem = true;
8505  // Call the Eigenproblem for the ajoint-problem eigensolver
8506  // NB Only different to solve_eigenproblem
8508  this, n_eval, eigenvalue, eigenvector, do_adjoint_problem);
8509 
8510  // Reset the is_steady status of all timesteppers that
8511  // weren't already steady when we came in here and reset their
8512  // weights
8513  for (unsigned i = 0; i < n_time_steppers; i++)
8514  {
8515  if (!was_steady[i])
8516  {
8518  }
8519  }
8520  }
8521  // Otherwise if we don't want to make the problem steady, just
8522  // assemble and solve the eigensystem
8523  else
8524  {
8525  const bool do_adjoint_problem = true;
8526  // Call the Eigenproblem for the eigensolver
8528  this, n_eval, eigenvalue, eigenvector, do_adjoint_problem);
8529  }
8530  }
8531 
8532 
8533  //==================================================================
8534  /// Solve the adjoint eigenproblem
8535  //==================================================================
8537  const unsigned& n_eval,
8538  Vector<std::complex<double>>& eigenvalue,
8539  Vector<DoubleVector>& eigenvector_real,
8540  Vector<DoubleVector>& eigenvector_imag,
8541  const bool& make_timesteppers_steady)
8542  {
8543  // If the boolean flag is steady, then make all the timesteppers steady
8544  // before solving the eigenproblem. This will "switch off" the
8545  // time-derivative terms in the jacobian matrix
8546  if (make_timesteppers_steady)
8547  {
8548  // Find out how many timesteppers there are
8549  const unsigned n_time_steppers = ntime_stepper();
8550 
8551  // Vector of bools to store the is_steady status of the various
8552  // timesteppers when we came in here
8553  std::vector<bool> was_steady(n_time_steppers);
8554 
8555  // Loop over them all and make them (temporarily) static
8556  for (unsigned i = 0; i < n_time_steppers; i++)
8557  {
8558  was_steady[i] = time_stepper_pt(i)->is_steady();
8560  }
8561 
8562  const bool do_adjoint_problem = true;
8563  // Call the Eigenproblem for the eigensolver
8565  n_eval,
8566  eigenvalue,
8567  eigenvector_real,
8568  eigenvector_imag,
8569  do_adjoint_problem);
8570 
8571  // Reset the is_steady status of all timesteppers that
8572  // weren't already steady when we came in here and reset their
8573  // weights
8574  for (unsigned i = 0; i < n_time_steppers; i++)
8575  {
8576  if (!was_steady[i])
8577  {
8579  }
8580  }
8581  }
8582  // Otherwise if we don't want to make the problem steady, just
8583  // assemble and solve the eigensystem
8584  else
8585  {
8586  const bool do_adjoint_problem = true;
8587  // Call the Eigenproblem for the eigensolver
8589  n_eval,
8590  eigenvalue,
8591  eigenvector_real,
8592  eigenvector_imag,
8593  do_adjoint_problem);
8594  }
8595  }
8596 
8597  //===================================================================
8598  /// Get the matrices required to solve an eigenproblem
8599  /// WARNING: temporarily this method only works with non-distributed
8600  /// matrices
8601  //===================================================================
8603  CRDoubleMatrix& main_matrix,
8604  const double& shift)
8605  {
8606  // Three different cases again here:
8607  // 1) Compiled with MPI, but run in serial
8608  // 2) Compiled with MPI, but MPI not initialised in driver
8609  // 3) Serial version
8610 
8611 
8612 #ifdef PARANOID
8613  if (mass_matrix.distribution_built() && main_matrix.distribution_built())
8614  {
8615  // Check that the distribution of the mass matrix and jacobian match
8616  if (!(*mass_matrix.distribution_pt() == *main_matrix.distribution_pt()))
8617  {
8618  std::ostringstream error_stream;
8619  error_stream
8620  << "The distributions of the jacobian and mass matrix are\n"
8621  << "not the same and they must be.\n";
8622  throw OomphLibError(
8623  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
8624  }
8625 
8626  if (mass_matrix.nrow() != this->ndof())
8627  {
8628  std::ostringstream error_stream;
8629  error_stream
8630  << "mass_matrix has a distribution, but the number of rows is not "
8631  << "equal to the number of degrees of freedom in the problem.";
8632  throw OomphLibError(
8633  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
8634  }
8635 
8636  if (main_matrix.nrow() != this->ndof())
8637  {
8638  std::ostringstream error_stream;
8639  error_stream
8640  << "main_matrix has a distribution, but the number of rows is not "
8641  << "equal to the number of degrees of freedom in the problem.";
8642  throw OomphLibError(
8643  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
8644  }
8645  }
8646  // If the distributions are not the same, then complain
8647  else if (main_matrix.distribution_built() !=
8648  mass_matrix.distribution_built())
8649  {
8650  std::ostringstream error_stream;
8651  error_stream << "The distribution of the jacobian and mass matrix must "
8652  << "both be setup or both not setup";
8653  throw OomphLibError(
8654  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
8655  }
8656 #endif
8657 
8658  // Store the old assembly handler
8659  AssemblyHandler* old_assembly_handler_pt = Assembly_handler_pt;
8660  // Now setup the eigenproblem handler, pass in the value of the shift
8662 
8663  // Prepare the storage formats.
8664  Vector<int*> column_or_row_index(2);
8665  Vector<int*> row_or_column_start(2);
8666  Vector<double*> value(2);
8667  Vector<unsigned> nnz(2);
8668  // Allocate pointer to residuals, although not used in these problems
8669  Vector<double*> residuals_vectors(0);
8670 
8671  // determine the distribution for the jacobian (main matrix)
8672  // IF the jacobian has distribution setup then use that
8673  // ELSE determine the distribution based on the
8674  // distributed_matrix_distribution enum
8675  LinearAlgebraDistribution* dist_pt = 0;
8676  if (main_matrix.distribution_built())
8677  {
8678  dist_pt = new LinearAlgebraDistribution(main_matrix.distribution_pt());
8679  }
8680  else
8681  {
8683  }
8684 
8685 
8686  // The matrix is in compressed row format
8687  bool compressed_row_flag = true;
8688 
8689 #ifdef OOMPH_HAS_MPI
8690  //
8691  if (Communicator_pt->nproc() == 1)
8692  {
8693 #endif
8694 
8695  sparse_assemble_row_or_column_compressed(column_or_row_index,
8696  row_or_column_start,
8697  value,
8698  nnz,
8699  residuals_vectors,
8700  compressed_row_flag);
8701 
8702  // The main matrix is the first entry
8703  main_matrix.build(dist_pt);
8704  main_matrix.build_without_copy(dist_pt->nrow(),
8705  nnz[0],
8706  value[0],
8707  column_or_row_index[0],
8708  row_or_column_start[0]);
8709  // The mass matrix is the second entry
8710  mass_matrix.build(dist_pt);
8711  mass_matrix.build_without_copy(dist_pt->nrow(),
8712  nnz[1],
8713  value[1],
8714  column_or_row_index[1],
8715  row_or_column_start[1]);
8716 #ifdef OOMPH_HAS_MPI
8717  }
8718  else
8719  {
8720  if (dist_pt->distributed())
8721  {
8722  parallel_sparse_assemble(dist_pt,
8723  column_or_row_index,
8724  row_or_column_start,
8725  value,
8726  nnz,
8727  residuals_vectors);
8728  // The main matrix is the first entry
8729  main_matrix.build(dist_pt);
8730  main_matrix.build_without_copy(dist_pt->nrow(),
8731  nnz[0],
8732  value[0],
8733  column_or_row_index[0],
8734  row_or_column_start[0]);
8735  // The mass matrix is the second entry
8736  mass_matrix.build(dist_pt);
8737  mass_matrix.build_without_copy(dist_pt->nrow(),
8738  nnz[1],
8739  value[1],
8740  column_or_row_index[1],
8741  row_or_column_start[1]);
8742  }
8743  else
8744  {
8745  LinearAlgebraDistribution* temp_dist_pt =
8746  new LinearAlgebraDistribution(Communicator_pt, dist_pt->nrow(), true);
8747  parallel_sparse_assemble(temp_dist_pt,
8748  column_or_row_index,
8749  row_or_column_start,
8750  value,
8751  nnz,
8752  residuals_vectors);
8753  // The main matrix is the first entry
8754  main_matrix.build(temp_dist_pt);
8755  main_matrix.build_without_copy(dist_pt->nrow(),
8756  nnz[0],
8757  value[0],
8758  column_or_row_index[0],
8759  row_or_column_start[0]);
8760  main_matrix.redistribute(dist_pt);
8761  // The mass matrix is the second entry
8762  mass_matrix.build(temp_dist_pt);
8763  mass_matrix.build_without_copy(dist_pt->nrow(),
8764  nnz[1],
8765  value[1],
8766  column_or_row_index[1],
8767  row_or_column_start[1]);
8768  mass_matrix.redistribute(dist_pt);
8769  delete temp_dist_pt;
8770  }
8771  }
8772 #endif
8773 
8774  // clean up dist_pt and residuals_vector pt
8775  delete dist_pt;
8776 
8777  // Delete the eigenproblem handler
8778  delete Assembly_handler_pt;
8779  // Reset the assembly handler to the original handler
8780  Assembly_handler_pt = old_assembly_handler_pt;
8781  }
8782 
8783 
8784  //=======================================================================
8785  /// Stored the current values of the dofs
8786  //=======================================================================
8788  {
8789  // If memory has not been allocated, then allocated memory for the saved
8790  // dofs
8791  if (Saved_dof_pt == 0)
8792  {
8794  }
8795 
8796 #ifdef OOMPH_HAS_MPI
8797  // If the problem is distributed I have to do something different
8799  {
8800  // How many entries do we store locally?
8801  const unsigned n_row_local = Dof_distribution_pt->nrow_local();
8802 
8803  // Resize the vector
8804  Saved_dof_pt->resize(n_row_local);
8805 
8806  // Back 'em up
8807  for (unsigned i = 0; i < n_row_local; i++)
8808  {
8809  (*Saved_dof_pt)[i] = *(this->Dof_pt[i]);
8810  }
8811  }
8812  // Otherwise just store all the dofs
8813  else
8814 #endif
8815  {
8816  // Find the number of dofs
8817  unsigned long n_dof = ndof();
8818 
8819  // Resize the vector
8820  Saved_dof_pt->resize(n_dof);
8821 
8822  // Transfer the values over
8823  for (unsigned long n = 0; n < n_dof; n++)
8824  {
8825  (*Saved_dof_pt)[n] = dof(n);
8826  }
8827  }
8828  }
8829 
8830  //====================================================================
8831  /// Restore the saved dofs
8832  //====================================================================
8834  {
8835  // Check that we can do this
8836  if (Saved_dof_pt == 0)
8837  {
8838  throw OomphLibError(
8839  "There are no stored values, use store_current_dof_values()\n",
8840  OOMPH_CURRENT_FUNCTION,
8841  OOMPH_EXCEPTION_LOCATION);
8842  }
8843 
8844 
8845 #ifdef OOMPH_HAS_MPI
8846  // If the problem is distributed I have to do something different
8848  {
8849  // How many entries do we store locally?
8850  const unsigned n_row_local = Dof_distribution_pt->nrow_local();
8851 
8852  if (Saved_dof_pt->size() != n_row_local)
8853  {
8854  throw OomphLibError("The number of stored values is not equal to the "
8855  "current number of dofs\n",
8856  OOMPH_CURRENT_FUNCTION,
8857  OOMPH_EXCEPTION_LOCATION);
8858  }
8859 
8860  // Transfer the values over
8861  for (unsigned long n = 0; n < n_row_local; n++)
8862  {
8863  *(this->Dof_pt[n]) = (*Saved_dof_pt)[n];
8864  }
8865  }
8866  // Otherwise just restore all the dofs
8867  else
8868 #endif
8869  {
8870  // Find the number of dofs
8871  unsigned long n_dof = ndof();
8872 
8873  if (Saved_dof_pt->size() != n_dof)
8874  {
8875  throw OomphLibError("The number of stored values is not equal to the "
8876  "current number of dofs\n",
8877  OOMPH_CURRENT_FUNCTION,
8878  OOMPH_EXCEPTION_LOCATION);
8879  }
8880 
8881  // Transfer the values over
8882  for (unsigned long n = 0; n < n_dof; n++)
8883  {
8884  dof(n) = (*Saved_dof_pt)[n];
8885  }
8886  }
8887 
8888  // Delete the memory
8889  delete Saved_dof_pt;
8890  Saved_dof_pt = 0;
8891  }
8892 
8893  //======================================================================
8894  /// Assign the eigenvector passed to the function to the dofs
8895  //======================================================================
8897  {
8898  unsigned long n_dof = ndof();
8899  // Check that the eigenvector has the correct size
8900  if (eigenvector.nrow() != n_dof)
8901  {
8902  std::ostringstream error_message;
8903  error_message << "Eigenvector has size " << eigenvector.nrow()
8904  << ", not equal to the number of dofs in the problem,"
8905  << n_dof << std::endl;
8906 
8907  throw OomphLibError(
8908  error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
8909  }
8910 
8911  // Ensure that the eigenvector distribution matches the dof distribution
8912  // Copy vector
8913  DoubleVector eigenvector_dof = eigenvector;
8914  // Redistribute the copy to the dof distribution
8915  eigenvector_dof.redistribute(this->Dof_distribution_pt);
8916 
8917  // Loop over the dofs and assign the eigenvector
8918  for (unsigned long n = 0; n < eigenvector_dof.nrow_local(); n++)
8919  {
8920  dof(n) = eigenvector_dof[n];
8921  }
8922 // Of course we now need to synchronise
8923 #ifdef OOMPH_HAS_MPI
8924  this->synchronise_all_dofs();
8925 #endif
8926  }
8927 
8928 
8929  //======================================================================
8930  /// Add the eigenvector passed to the function to the dofs with
8931  /// magnitude epsilon
8932  //======================================================================
8933  void Problem::add_eigenvector_to_dofs(const double& epsilon,
8934  const DoubleVector& eigenvector)
8935  {
8936  unsigned long n_dof = ndof();
8937  // Check that the eigenvector has the correct size
8938  if (eigenvector.nrow() != n_dof)
8939  {
8940  std::ostringstream error_message;
8941  error_message << "Eigenvector has size " << eigenvector.nrow()
8942  << ", not equal to the number of dofs in the problem,"
8943  << n_dof << std::endl;
8944 
8945  throw OomphLibError(
8946  error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
8947  }
8948 
8949  // Ensure that the eigenvector distribution matches the dof distribution
8950  // Copy vector
8951  DoubleVector eigenvector_dof = eigenvector;
8952  // Redistribute the copy to the dof distribution
8953  eigenvector_dof.redistribute(this->Dof_distribution_pt);
8954 
8955 
8956  // Loop over the dofs and add the eigenvector
8957  // Only use local values
8958  for (unsigned long n = 0; n < eigenvector.nrow_local(); n++)
8959  {
8960  dof(n) += epsilon * eigenvector[n];
8961  }
8962 // Of course we now need to synchronise
8963 #ifdef OOMPH_HAS_MPI
8964  this->synchronise_all_dofs();
8965 #endif
8966  }
8967 
8968 
8969  //================================================================
8970  /// General Newton solver. Requires only a convergence tolerance.
8971  /// The linear solver takes a pointer to the problem (which defines
8972  /// the Jacobian \b J and the residual Vector \b r) and returns
8973  /// the solution \b x of the system
8974  /// \f[ {\bf J} {\bf x} = - \bf{r} \f].
8975  //================================================================
8977  {
8978  // Initialise timers
8979  double total_linear_solver_time = 0.0;
8980  double t_start = TimingHelpers::timer();
8981  Max_res.clear();
8982 
8983  // Find total number of dofs
8984  unsigned long n_dofs = ndof();
8985 
8986  // Set up the Vector to hold the solution
8987  DoubleVector dx;
8988 
8989  //-----Variables for the globally convergent Newton method------
8990 
8991  // Set up the vector to hold the gradient
8992  DoubleVector gradient;
8993 
8994  // Other variables
8995  double half_residual_squared = 0.0;
8996  double max_step = 0.0;
8997 
8998  //--------------------------------------------------------------
8999 
9000  // Set the counter
9001  unsigned count = 0;
9002  // Set the loop flag
9003  unsigned LOOP_FLAG = 1;
9004 
9006  {
9007 #ifdef OOMPH_HAS_MPI
9008  // Break if running in parallel
9010  {
9011  std::ostringstream error_stream;
9012  error_stream << "Globally convergent Newton method has not been "
9013  << "implemented in parallel yet!" << std::endl;
9014  throw OomphLibError(
9015  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
9016  }
9017 #endif
9018 
9019  // Get gradient
9021  // Reset the gradient (clear it), since the number of dofs and
9022  // hence the size of the DoubleVector might have changed
9024  }
9025 
9026  // Update anything that needs updating
9028 
9029  // Reset number of Newton iterations taken
9030  Nnewton_iter_taken = 0;
9031 
9032  // Now do the Newton loop
9033  do
9034  {
9035  count++;
9036 
9037  // Do any updates that are required
9039 
9040 
9041  // No degrees of freedom? What are you solving for?
9042  if (n_dofs == 0)
9043  {
9044  oomph_info << std::endl << std::endl << std::endl;
9045  oomph_info << "This is a bit bizarre: The problem has no dofs."
9046  << std::endl;
9047  oomph_info
9048  << "I'll just return from the Newton solver without doing anything."
9049  << std::endl;
9050 
9051  // Do any updates that would have been performed
9056 
9057  oomph_info << "I hope this is what you intended me to do..."
9058  << std::endl;
9059  oomph_info << std::endl
9060  << "Note: All actions_...() functions were called"
9061  << std::endl;
9062  oomph_info << std::endl << " before returning." << std::endl;
9063  oomph_info << std::endl << std::endl << std::endl;
9064  return;
9065  }
9066 
9067  // Calculate initial residuals
9068  if (count == 1)
9069  {
9070  // Is the problem nonlinear? If not ignore the pre-iteration
9071  // convergence check.
9073  {
9074 #ifdef OOMPH_HAS_MPI
9075  // Synchronise the solution on different processors (on each submesh)
9076  this->synchronise_all_dofs();
9077 #endif
9078 
9080  dx.clear();
9081  get_residuals(dx);
9082 
9083  // Get half of squared residual and find maximum step length
9084  // for step length control
9086  {
9087  half_residual_squared = 0.0;
9088  double sum = 0.0;
9089  for (unsigned i = 0; i < n_dofs; i++)
9090  {
9091  sum += (*Dof_pt[i]) * (*Dof_pt[i]);
9092  half_residual_squared += dx[i] * dx[i];
9093  }
9094  half_residual_squared *= 0.5;
9095  max_step = 100.0 * std::max(sqrt(sum), double(n_dofs));
9096  }
9097 
9098  // Get maximum residuals
9099  double maxres = dx.max();
9100  Max_res.push_back(maxres);
9101 
9103  {
9104  // Let's output the residuals
9105  // unsigned n_row_local = dx.distribution_pt()->nrow_local();
9106  // unsigned first_row = dx.distribution_pt()->first_row();
9107  // for(unsigned n=0;n<n_row_local;n++)
9108  //{
9109  // oomph_info << "residual: " << n + first_row << " " << dx[n] <<
9110  // "\n";
9111  //}
9112 
9113  oomph_info << "\nInitial Maximum residuals " << maxres << std::endl;
9114  }
9115 
9116  if ((maxres < Newton_solver_tolerance) &&
9118  {
9119  LOOP_FLAG = 0;
9120  continue;
9121  }
9122  }
9123  else
9124  {
9126  {
9127  oomph_info
9128  << "Linear problem -- convergence in one iteration assumed."
9129  << std::endl;
9130  }
9131  }
9132  }
9133 
9134 
9135  // Increment number of Newton iterations taken
9137 
9138  // Initialise timer for linear solver
9139  double t_solver_start = TimingHelpers::timer();
9140 
9141  // Now do the linear solve -- recycling Jacobian if requested
9143  {
9145  {
9146  oomph_info << "Not recomputing Jacobian! " << std::endl;
9147  }
9148 
9149  // If we're doing the first iteration and the problem is nonlinear,
9150  // the residuals have already been computed above during the
9151  // initial convergence check. Otherwise compute them here.
9152  if ((count != 1) || (!Problem_is_nonlinear)) get_residuals(dx);
9153 
9154  // Backup residuals
9155  DoubleVector resid(dx);
9156 
9157  // Resolve
9158  Linear_solver_pt->resolve(resid, dx);
9159  }
9160  else
9161  {
9163  {
9165  {
9166  oomph_info << "Enabling resolve" << std::endl;
9167  }
9169  }
9170  Linear_solver_pt->solve(this, dx);
9172  }
9173 
9174  // End of linear solver
9175  double t_solver_end = TimingHelpers::timer();
9176  total_linear_solver_time += t_solver_end - t_solver_start;
9177 
9179  {
9180  oomph_info << std::endl;
9181  oomph_info << "Time for linear solver (ndof=" << n_dofs << "): "
9183  t_solver_end - t_solver_start)
9184  << std::endl
9185  << std::endl;
9186  }
9187 
9188  // Subtract the new values from the true dofs
9190  double* dx_pt = dx.values_pt();
9191  unsigned ndof_local = Dof_distribution_pt->nrow_local();
9192 
9194  {
9195  // Get the gradient
9196  Linear_solver_pt->get_gradient(gradient);
9197 
9198  for (unsigned i = 0; i < ndof_local; i++)
9199  {
9200  dx_pt[i] *= -1.0;
9201  }
9202 
9203  // Update with steplength control
9204  Vector<double> unknowns_old(ndof_local);
9205 
9206  for (unsigned i = 0; i < ndof_local; i++)
9207  {
9208  unknowns_old[i] = *Dof_pt[i];
9209  }
9210 
9211  double half_residual_squared_old = half_residual_squared;
9212  globally_convergent_line_search(unknowns_old,
9213  half_residual_squared_old,
9214  gradient,
9215  dx,
9216  half_residual_squared,
9217  max_step);
9218  }
9219  // direct Newton update
9220  else
9221  {
9222  for (unsigned l = 0; l < ndof_local; l++)
9223  {
9224  *Dof_pt[l] -= Relaxation_factor * dx_pt[l];
9225  }
9226  }
9227 #ifdef OOMPH_HAS_MPI
9228  // Synchronise the solution on different processors (on each submesh)
9229  this->synchronise_all_dofs();
9230 #endif
9231 
9232  // Do any updates that are required
9235 
9236  // Maximum residuals
9237  double maxres = 0.0;
9238  // If the user has declared that the Problem is linear
9239  // we ignore the convergence check
9241  {
9242  // Get the maximum residuals
9243  // maxres = std::fabs(*std::max_element(dx.begin(),dx.end(),
9244  // AbsCmp<double>()));
9245  // oomph_info << "Maxres correction " << maxres << "\n";
9246 
9247  // Calculate the new residuals
9248  dx.clear();
9249  get_residuals(dx);
9250 
9251  // Get the maximum residuals
9252  maxres = dx.max();
9253  Max_res.push_back(maxres);
9254 
9256  {
9257  oomph_info << "Newton Step " << count << ": Maximum residuals "
9258  << maxres << std::endl
9259  << std::endl;
9260  }
9261  }
9262 
9263  // If we have converged jump straight to the test at the end of the loop
9264  if (maxres < Newton_solver_tolerance)
9265  {
9266  LOOP_FLAG = 0;
9267  continue;
9268  }
9269 
9270  // This section will not be reached if we have converged already
9271  // If the maximum number of residuals is too high or the maximum number
9272  // of iterations has been reached
9273  if ((maxres > Max_residuals) || (count == Max_newton_iterations))
9274  {
9275  // Print a warning -- regardless of what the throw does
9276  if (maxres > Max_residuals)
9277  {
9278  oomph_info << "Max. residual (" << Max_residuals
9279  << ") has been exceeded in Newton solver." << std::endl;
9280  }
9281  if (count == Max_newton_iterations)
9282  {
9283  oomph_info << "Reached max. number of iterations ("
9284  << Max_newton_iterations << ") in Newton solver."
9285  << std::endl;
9286  }
9287  // Now throw...
9288  throw NewtonSolverError(count, maxres);
9289  }
9290 
9291  } while (LOOP_FLAG);
9292 
9293  // Now update anything that needs updating
9295 
9296  // Finalise/doc timings
9298  {
9299  oomph_info << std::endl;
9300  oomph_info << "Total time for linear solver (ndof=" << n_dofs << "): "
9302  total_linear_solver_time)
9303  << std::endl;
9304  }
9305 
9306  double t_end = TimingHelpers::timer();
9307  double total_time = t_end - t_start;
9308 
9310  {
9311  oomph_info << "Total time for Newton solver (ndof=" << n_dofs << "): "
9313  << std::endl;
9314  }
9315  if (total_time > 0.0)
9316  {
9318  {
9319  oomph_info << "Time outside linear solver : "
9320  << (total_time - total_linear_solver_time) / total_time *
9321  100.0
9322  << " %" << std::endl;
9323  }
9324  }
9325  else
9326  {
9328  {
9329  oomph_info << "Time outside linear solver : "
9330  << "[too fast]" << std::endl;
9331  }
9332  }
9333  if (!Shut_up_in_newton_solve) oomph_info << std::endl;
9334  }
9335 
9336  //========================================================================
9337  /// Helper function for the globally convergent Newton solver
9338  //========================================================================
9340  const Vector<double>& x_old,
9341  const double& half_residual_squared_old,
9342  DoubleVector& gradient,
9343  DoubleVector& newton_dir,
9344  double& half_residual_squared,
9345  const double& stpmax)
9346  {
9347  const double min_fct_decrease = 1.0e-4;
9348  double convergence_tol_on_x = 1.0e-16;
9349  double f_aux = 0.0;
9350  double lambda_aux = 0.0;
9351  double proposed_lambda;
9352  unsigned long n_dof = ndof();
9353  double sum = 0.0;
9354  for (unsigned i = 0; i < n_dof; i++)
9355  {
9356  sum += newton_dir[i] * newton_dir[i];
9357  }
9358  sum = sqrt(sum);
9359  if (sum > stpmax)
9360  {
9361  for (unsigned i = 0; i < n_dof; i++)
9362  {
9363  newton_dir[i] *= stpmax / sum;
9364  }
9365  }
9366  double slope = 0.0;
9367  for (unsigned i = 0; i < n_dof; i++)
9368  {
9369  slope += gradient[i] * newton_dir[i];
9370  }
9371  if (slope >= 0.0)
9372  {
9373  std::ostringstream warn_message;
9374  warn_message << "WARNING: Non-negative slope, probably due to a "
9375  << " roundoff \nproblem in the linesearch: slope=" << slope
9376  << "\n";
9377  OomphLibWarning(warn_message.str(),
9378  "Problem::globally_convergent_line_search()",
9379  OOMPH_EXCEPTION_LOCATION);
9380  }
9381  double test = 0.0;
9382  for (unsigned i = 0; i < n_dof; i++)
9383  {
9384  double temp =
9385  std::fabs(newton_dir[i]) / std::max(std::fabs(x_old[i]), 1.0);
9386  if (temp > test) test = temp;
9387  }
9388  double lambda_min = convergence_tol_on_x / test;
9389  double lambda = 1.0;
9390  while (true)
9391  {
9392  for (unsigned i = 0; i < n_dof; i++)
9393  {
9394  *Dof_pt[i] = x_old[i] + lambda * newton_dir[i];
9395  }
9396 
9397  // Evaluate current residuals
9398  DoubleVector residuals;
9399  get_residuals(residuals);
9400  half_residual_squared = 0.0;
9401  for (unsigned i = 0; i < n_dof; i++)
9402  {
9403  half_residual_squared += residuals[i] * residuals[i];
9404  }
9405  half_residual_squared *= 0.5;
9406 
9407  if (lambda < lambda_min)
9408  {
9409  for (unsigned i = 0; i < n_dof; i++) *Dof_pt[i] = x_old[i];
9410 
9411  std::ostringstream warn_message;
9412  warn_message << "WARNING: Line search converged on x only!\n";
9413  OomphLibWarning(warn_message.str(),
9414  "Problem::globally_convergent_line_search()",
9415  OOMPH_EXCEPTION_LOCATION);
9416  return;
9417  }
9418  else if (half_residual_squared <=
9419  half_residual_squared_old + min_fct_decrease * lambda * slope)
9420  {
9421  oomph_info << "Returning from linesearch with lambda=" << lambda
9422  << std::endl;
9423  return;
9424  }
9425  else
9426  {
9427  if (lambda == 1.0)
9428  {
9429  proposed_lambda =
9430  -slope /
9431  (2.0 * (half_residual_squared - half_residual_squared_old - slope));
9432  }
9433  else
9434  {
9435  double r1 =
9436  half_residual_squared - half_residual_squared_old - lambda * slope;
9437  double r2 = f_aux - half_residual_squared_old - lambda_aux * slope;
9438  double a_poly =
9439  (r1 / (lambda * lambda) - r2 / (lambda_aux * lambda_aux)) /
9440  (lambda - lambda_aux);
9441  double b_poly = (-lambda_aux * r1 / (lambda * lambda) +
9442  lambda * r2 / (lambda_aux * lambda_aux)) /
9443  (lambda - lambda_aux);
9444  if (a_poly == 0.0)
9445  {
9446  proposed_lambda = -slope / (2.0 * b_poly);
9447  }
9448  else
9449  {
9450  double discriminant = b_poly * b_poly - 3.0 * a_poly * slope;
9451  if (discriminant < 0.0)
9452  {
9453  proposed_lambda = 0.5 * lambda;
9454  }
9455  else if (b_poly <= 0.0)
9456  {
9457  proposed_lambda = (-b_poly + sqrt(discriminant)) / (3.0 * a_poly);
9458  }
9459  else
9460  {
9461  proposed_lambda = -slope / (b_poly + sqrt(discriminant));
9462  }
9463  }
9464  if (proposed_lambda > 0.5 * lambda)
9465  {
9466  proposed_lambda = 0.5 * lambda;
9467  }
9468  }
9469  }
9470  lambda_aux = lambda;
9471  f_aux = half_residual_squared;
9472  lambda = std::max(proposed_lambda, 0.1 * lambda);
9473  }
9474  }
9475 
9476 
9477  //========================================================================
9478  /// Solve a steady problem, in the context of an overall unsteady problem.
9479  /// This is achieved by setting the weights in the timesteppers to be zero
9480  /// which has the effect of rendering them steady timesteppers
9481  /// The optional argument max_adapt specifies the max. number of
9482  /// adaptations of all refineable submeshes are performed to
9483  /// achieve the the error targets specified in the refineable submeshes.
9484  //========================================================================
9485  void Problem::steady_newton_solve(unsigned const& max_adapt)
9486  {
9487  // Find out how many timesteppers there are
9488  unsigned n_time_steppers = ntime_stepper();
9489 
9490  // Vector of bools to store the is_steady status of the various
9491  // timesteppers when we came in here
9492  std::vector<bool> was_steady(n_time_steppers);
9493 
9494  // Loop over them all and make them (temporarily) static
9495  for (unsigned i = 0; i < n_time_steppers; i++)
9496  {
9497  was_steady[i] = time_stepper_pt(i)->is_steady();
9499  }
9500 
9501  try
9502  {
9503  // Solve the non-linear problem with Newton's method
9504  if (max_adapt == 0)
9505  {
9506  newton_solve();
9507  }
9508  else
9509  {
9510  newton_solve(max_adapt);
9511  }
9512  }
9513  // Catch any exceptions thrown in the Newton solver
9514  catch (NewtonSolverError& error)
9515  {
9516  oomph_info << std::endl
9517  << "USER-DEFINED ERROR IN NEWTON SOLVER " << std::endl;
9518  // Check whether it's the linear solver
9519  if (error.linear_solver_error)
9520  {
9521  oomph_info << "ERROR IN THE LINEAR SOLVER" << std::endl;
9522  }
9523  // Check to see whether we have reached Max_iterations
9524  else if (error.iterations == Max_newton_iterations)
9525  {
9526  oomph_info << "MAXIMUM NUMBER OF ITERATIONS (" << error.iterations
9527  << ") REACHED WITHOUT CONVERGENCE " << std::endl;
9528  }
9529  // If not, it must be that we have exceeded the maximum residuals
9530  else
9531  {
9532  oomph_info << "MAXIMUM RESIDUALS: " << error.maxres
9533  << " EXCEEDS PREDEFINED MAXIMUM " << Max_residuals
9534  << std::endl;
9535  }
9536 
9537  // Die horribly!!
9538  std::ostringstream error_stream;
9539  error_stream << "Error occured in Newton solver. " << std::endl;
9540  throw OomphLibError(
9541  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
9542  }
9543 
9544 
9545  // Reset the is_steady status of all timesteppers that
9546  // weren't already steady when we came in here and reset their
9547  // weights
9548  for (unsigned i = 0; i < n_time_steppers; i++)
9549  {
9550  if (!was_steady[i])
9551  {
9553  }
9554  }
9555 
9556  // Since we performed a steady solve, the history values
9557  // now have to be set as if we had performed an impulsive start from
9558  // the current solution. This ensures that the time-derivatives
9559  // evaluate to zero even now that the timesteppers have been
9560  // reactivated.
9562  }
9563 
9564  //===========================================================================
9565  /// Perform a basic continuation step using Newton's method. The governing
9566  /// parameter of the problem is passed as a pointer to the routine. The
9567  /// number of Newton steps taken is returned
9568  //==========================================================================
9569  unsigned Problem::newton_solve_continuation(double* const& parameter_pt)
9570  {
9571  // Set up memory for z
9572  // unsigned long n_dofs = ndof();
9573  // LinearAlgebraDistribution dist(Communicator_pt,n_dofs,false);
9574  // DoubleVector z(&dist,0.0);
9575  DoubleVector z;
9576  // Call the solver
9577  return newton_solve_continuation(parameter_pt, z);
9578  }
9579 
9580 
9581  //===================================================================
9582  /// This function performs a basic continuation step using the Newton method.
9583  /// The number of Newton steps taken is returned, to be used in any
9584  /// external step-size control routines.
9585  /// The governing parameter of the problem is passed as a pointer to the
9586  /// routine, as is the sign of the Jacobian and a Vector in which
9587  /// to store the derivatives wrt the parameter, if required.
9588  //==================================================================
9589  unsigned Problem::newton_solve_continuation(double* const& parameter_pt,
9590  DoubleVector& z)
9591  {
9592  // Find the total number of dofs
9593  // unsigned long n_dofs = ndof();
9594 
9595  // Find the local number of dofs
9596  unsigned ndof_local = Dof_distribution_pt->nrow_local();
9597 
9598  // create the distribution (not distributed)
9599  // LinearAlgebraDistribution dist(this->communicator_pt(),n_dofs,false);
9600 
9601  // Assign memory for solutions of the equations
9602  // DoubleVector y(&dist,0.0);
9603  DoubleVector y;
9604 
9605  // Assign memory for the dot products of the uderivatives and y and z
9606  double uderiv_dot_y = 0.0, uderiv_dot_z = 0.0;
9607  // Set and initialise the counter
9608  unsigned count = 0;
9609  // Set the loop flag
9610  unsigned LOOP_FLAG = 1;
9611 
9612  // Update anything that needs updating
9614 
9615  // Check the arc-length constraint
9616  double arc_length_constraint_residual = 0.0;
9617 
9618  // Are we storing the matrix in the linear solve
9619  bool enable_resolve = Linear_solver_pt->is_resolve_enabled();
9620 
9621  // For this problem, we must store the residuals
9623 
9624  // Now do the Newton loop
9625  do
9626  {
9627  count++;
9628 
9629  // Do any updates that are required
9631 
9632  // Calculate initial residuals
9633  if (count == 1)
9634  {
9635 #ifdef OOMPH_HAS_MPI
9636  // Synchronise the solution on different processors (on each submesh)
9637  this->synchronise_all_dofs();
9638 #endif
9639 
9641  y.clear();
9642  get_residuals(y);
9643  // Get maximum residuals, using our own abscmp function
9644  double maxres = y.max();
9645 
9646  // Assemble the residuals for the arc-length step
9647  arc_length_constraint_residual = 0.0;
9648  // Add the variables
9649  for (unsigned long l = 0; l < ndof_local; l++)
9650  {
9651  arc_length_constraint_residual +=
9652  dof_derivative(l) * (*Dof_pt[l] - dof_current(l));
9653  }
9654 
9655  // Now reduce if we have been distributed
9656 #ifdef OOMPH_HAS_MPI
9657  double arc_length_cons_res2 = arc_length_constraint_residual;
9658  if ((Dof_distribution_pt->distributed()) &&
9659  (Dof_distribution_pt->communicator_pt()->nproc() > 1))
9660  {
9661  MPI_Allreduce(&arc_length_constraint_residual,
9662  &arc_length_cons_res2,
9663  1,
9664  MPI_DOUBLE,
9665  MPI_SUM,
9666  Dof_distribution_pt->communicator_pt()->mpi_comm());
9667  }
9668  arc_length_constraint_residual = arc_length_cons_res2;
9669 #endif
9670 
9671  arc_length_constraint_residual *= Theta_squared;
9672  arc_length_constraint_residual +=
9673  Parameter_derivative * (*parameter_pt - Parameter_current) -
9674  Ds_current;
9675 
9676  // Is it the max
9677  if (std::fabs(arc_length_constraint_residual) > maxres)
9678  {
9679  maxres = std::fabs(arc_length_constraint_residual);
9680  }
9681 
9682  // Find the max
9684  {
9685  oomph_info << "Initial Maximum residuals " << maxres << std::endl;
9686  }
9687 
9688  // If we are below the Tolerance, then return immediately
9689  if ((maxres < Newton_solver_tolerance) &&
9691  {
9692  LOOP_FLAG = 0;
9693  count = 0;
9694  continue;
9695  }
9696  }
9697 
9698  // If it's the block hopf solver we need to solve for both rhs's
9699  // simultaneously. This is because the block decomposition involves
9700  // solves with two different matrices and storing both at once to
9701  // allow general resolves would be more expensive than necessary.
9702  if (dynamic_cast<BlockHopfLinearSolver*>(Linear_solver_pt))
9703  {
9704  // Get the vector dresiduals/dparameter
9705  z.clear();
9706  get_derivative_wrt_global_parameter(parameter_pt, z);
9707 
9708  // Copy rhs vector into local storage so it doesn't get overwritten
9709  // if the linear solver decides to initialise the solution vector, say,
9710  // which it's quite entitled to do!
9711  DoubleVector input_z(z);
9712 
9713  // Solve the system for the two right-hand sides.
9714  dynamic_cast<BlockHopfLinearSolver*>(Linear_solver_pt)
9715  ->solve_for_two_rhs(this, y, input_z, z);
9716  }
9717  // Otherwise
9718  else
9719  {
9720  // Solve the standard problem
9721  Linear_solver_pt->solve(this, y);
9722 
9723  // Get the vector dresiduals/dparameter
9724  z.clear();
9725  get_derivative_wrt_global_parameter(parameter_pt, z);
9726 
9727  // Copy rhs vector into local storage so it doesn't get overwritten
9728  // if the linear solver decides to initialise the solution vector, say,
9729  // which it's quite entitled to do!
9730  DoubleVector input_z(z);
9731 
9732  // Redistribute the RHS to match the linear solver
9733  // input_z.redistribute(Linear_solver_pt->distribution_pt());
9734  // Do not clear z because we assume that it has dR/dparam
9735  z.clear();
9736  // Now resolve the system with the new RHS
9737  Linear_solver_pt->resolve(input_z, z);
9738  }
9739 
9740  // Redistribute the results into the natural distribution
9743 
9744  // Now we need to calculate dparam, for which we must calculate the
9745  // dot product of the derivatives and y and z
9746  // Reset these values to zero
9747  uderiv_dot_y = 0.0;
9748  uderiv_dot_z = 0.0;
9749  // Now calculate the dot products of the derivative and the solutions
9750  // of the linear system
9751  // Cache pointers to the data in the distributed vectors
9752  double* const y_pt = y.values_pt();
9753  double* const z_pt = z.values_pt();
9754  for (unsigned long l = 0; l < ndof_local; l++)
9755  {
9756  uderiv_dot_y += dof_derivative(l) * y_pt[l];
9757  uderiv_dot_z += dof_derivative(l) * z_pt[l];
9758  }
9759 
9760  // Now reduce if we have been distributed
9761 #ifdef OOMPH_HAS_MPI
9762  // Create send and receive arrays of size two
9763  double uderiv_dot[2];
9764  double uderiv_dot2[2];
9765  uderiv_dot[0] = uderiv_dot_y;
9766  uderiv_dot[1] = uderiv_dot_z;
9767  uderiv_dot2[0] = uderiv_dot_y;
9768  uderiv_dot2[1] = uderiv_dot_z;
9769  // Now reduce both together
9770  if ((Dof_distribution_pt->distributed()) &&
9771  (Dof_distribution_pt->communicator_pt()->nproc() > 1))
9772  {
9773  MPI_Allreduce(uderiv_dot,
9774  uderiv_dot2,
9775  2,
9776  MPI_DOUBLE,
9777  MPI_SUM,
9778  Dof_distribution_pt->communicator_pt()->mpi_comm());
9779  }
9780  uderiv_dot_y = uderiv_dot2[0];
9781  uderiv_dot_z = uderiv_dot2[1];
9782 #endif
9783 
9784  // Now scale the results
9785  uderiv_dot_y *= Theta_squared;
9786  uderiv_dot_z *= Theta_squared;
9787 
9788  // The set the change in the parameter, given by the pseudo-arclength
9789  // equation. Note that here we are assuming that the arc-length
9790  // equation is always exactly zero,
9791  // which seems to work OK, and saves on some storage.
9792  // In fact, it's more subtle than that. If we include this
9793  // proper residual then we will have to solve the eigenproblem.
9794  // This will make the solver more robust and *should* be done
9795  // ... at some point.
9796  double dparam = (arc_length_constraint_residual - uderiv_dot_y) /
9797  (Parameter_derivative - uderiv_dot_z);
9798 
9799  // Set the new value of the parameter
9800  *parameter_pt -= dparam;
9801 
9802  // Update the values of the other degrees of freedom
9803  for (unsigned long l = 0; l < ndof_local; l++)
9804  {
9805  *Dof_pt[l] -= y_pt[l] - dparam * z_pt[l];
9806  }
9807 
9808  // Calculate the new residuals
9809 #ifdef OOMPH_HAS_MPI
9810  // Synchronise the solution on different processors (on each submesh)
9811  this->synchronise_all_dofs();
9812 #endif
9813 
9814  // Do any updates that are required
9817 
9818  y.clear();
9819  get_residuals(y);
9820 
9821  // Get the maximum residuals
9822  double maxres = y.max();
9823 
9824  // Assemble the residuals for the arc-length step
9825  arc_length_constraint_residual = 0.0;
9826  // Add the variables
9827  for (unsigned long l = 0; l < ndof_local; l++)
9828  {
9829  arc_length_constraint_residual +=
9830  dof_derivative(l) * (*Dof_pt[l] - dof_current(l));
9831  }
9832 
9833  // Now reduce if we have been distributed
9834 #ifdef OOMPH_HAS_MPI
9835  double arc_length_cons_res2 = arc_length_constraint_residual;
9836  if ((Dof_distribution_pt->distributed()) &&
9837  (Dof_distribution_pt->communicator_pt()->nproc() > 1))
9838  {
9839  MPI_Allreduce(&arc_length_constraint_residual,
9840  &arc_length_cons_res2,
9841  1,
9842  MPI_DOUBLE,
9843  MPI_SUM,
9844  Dof_distribution_pt->communicator_pt()->mpi_comm());
9845  }
9846  arc_length_constraint_residual = arc_length_cons_res2;
9847 #endif
9848 
9849  arc_length_constraint_residual *= Theta_squared;
9850  arc_length_constraint_residual +=
9851  Parameter_derivative * (*parameter_pt - Parameter_current) - Ds_current;
9852 
9853  // Is it the max
9854  if (std::fabs(arc_length_constraint_residual) > maxres)
9855  {
9856  maxres = std::fabs(arc_length_constraint_residual);
9857  }
9858 
9860  {
9861  oomph_info << "Continuation Step " << count << ": Maximum residuals "
9862  << maxres << "\n";
9863  }
9864 
9865  // If we have converged jump straight to the test at the end of the loop
9866  if (maxres < Newton_solver_tolerance)
9867  {
9868  LOOP_FLAG = 0;
9869  continue;
9870  }
9871 
9872  // This section will not be reached if we have converged already
9873  // If the maximum number of residuals is too high or the maximum number
9874  // of iterations has been reached
9875  if ((maxres > Max_residuals) || (count == Max_newton_iterations))
9876  {
9877  throw NewtonSolverError(count, maxres);
9878  }
9879 
9880  } while (LOOP_FLAG);
9881 
9882  // Now update anything that needs updating
9884 
9885  // Reset the storage of the matrix on the linear solver to what it was
9886  // on entry to this routine
9887  if (enable_resolve)
9888  {
9890  }
9891  else
9892  {
9894  }
9895 
9896  // Return the number of Newton Steps taken
9897  return count;
9898  }
9899 
9900  //=========================================================================
9901  /// A function to calculate the derivatives wrt the arc-length. This version
9902  /// of the function actually does a linear solve so that the derivatives
9903  /// are calculated "exactly" rather than using the values at the Newton
9904  /// step just before convergence. This is only necessary in spatially adaptive
9905  /// problems, in which the number of degrees of freedom changes and so
9906  /// the appropriate derivatives must be calculated for the new variables.
9907  //=========================================================================
9908  void Problem::calculate_continuation_derivatives(double* const& parameter_pt)
9909  {
9910  // Find the number of degrees of freedom in the problem
9911  const unsigned long n_dofs = ndof();
9912 
9913  // create a non-distributed z vector
9914  LinearAlgebraDistribution dist(Communicator_pt, n_dofs, false);
9915 
9916  // Assign memory for solutions of the equations
9917  DoubleVector z(&dist, 0.0);
9918 
9919  // If it's the block hopf solver need to solve for both RHS
9920  // at once, but this would all be alleviated if we have the solve
9921  // for the non-residuals RHS.
9922  if (dynamic_cast<BlockHopfLinearSolver*>(Linear_solver_pt))
9923  {
9924  // Get the vector dresiduals/dparameter
9925  get_derivative_wrt_global_parameter(parameter_pt, z);
9926 
9927  // Copy rhs vector into local storage so it doesn't get overwritten
9928  // if the linear solver decides to initialise the solution vector, say,
9929  // which it's quite entitled to do!
9930  DoubleVector dummy(&dist, 0.0), input_z(z);
9931 
9932  // Solve for the two RHSs
9933  dynamic_cast<BlockHopfLinearSolver*>(Linear_solver_pt)
9934  ->solve_for_two_rhs(this, dummy, input_z, z);
9935  }
9936  // Otherwise we can use the normal resolve
9937  else
9938  {
9939  // Save the status before entry to this routine
9940  bool enable_resolve = Linear_solver_pt->is_resolve_enabled();
9941 
9942  // We need to do resolves
9944 
9945  // Solve the standard problem, we only want to make sure that
9946  // we factorise the matrix, if it has not been factorised. We shall
9947  // ignore the return value of z.
9948  Linear_solver_pt->solve(this, z);
9949 
9950  // Get the vector dresiduals/dparameter
9951  get_derivative_wrt_global_parameter(parameter_pt, z);
9952 
9953 
9954  // Copy rhs vector into local storage so it doesn't get overwritten
9955  // if the linear solver decides to initialise the solution vector, say,
9956  // which it's quite entitled to do!
9957  DoubleVector input_z(z);
9958 
9959  // Now resolve the system with the new RHS and overwrite the solution
9960  Linear_solver_pt->resolve(input_z, z);
9961 
9962  // Restore the storage status of the linear solver
9963  if (enable_resolve)
9964  {
9966  }
9967  else
9968  {
9970  }
9971  }
9972 
9973  // Now, we can calculate the derivatives, etc
9975  }
9976 
9977  //=======================================================================
9978  /// A function to calculate the derivatives with respect to the arc-length
9979  /// required for continuation. The arguments is the solution of the
9980  /// linear system,
9981  /// Jz = dR/dparameter, that gives du/dparameter and the direction
9982  /// output from the newton_solve_continuation function. The derivatives
9983  /// are stored in the ContinuationParameters namespace.
9984  //===================================================================
9986  {
9987  // Calculate the continuation derivatives
9989 
9990  // Scale the value of theta if the control flag is set
9991  if (Scale_arc_length)
9992  {
9993  // Don't divide by zero!
9994  if (Parameter_derivative != 1.0)
9995  {
10000 
10001  // Recalculate the continuation derivatives with the new scaled values
10003  }
10004  }
10005  }
10006 
10007  //=======================================================================
10008  /// A function to calculate the derivatives with respect to the arc-length
10009  /// required for continuation using finite differences.
10010  //===================================================================
10012  double* const& parameter_pt)
10013  {
10014  // Calculate the continuation derivatives
10016 
10017  // Scale the value of theta if the control flag is set
10018  if (Scale_arc_length)
10019  {
10020  // Don't divide by zero!
10021  if (Parameter_derivative != 1.0)
10022  {
10027 
10028  // Recalculate the continuation derivatives with the new scaled values
10030  }
10031  }
10032  }
10033 
10034  //======================================================================
10035  /// Function that returns a boolean flag to indicate whether the pointer
10036  /// parameter_pt refers to memory that is a value in a Data object used
10037  /// within the problem
10038  //======================================================================
10040  double* const& parameter_pt)
10041  {
10042  // Firstly check the global data
10043  const unsigned n_global = Global_data_pt.size();
10044  for (unsigned i = 0; i < n_global; ++i)
10045  {
10046  // If we find it then return true
10047  if (Global_data_pt[i]->does_pointer_correspond_to_value(parameter_pt))
10048  {
10049  return true;
10050  }
10051  }
10052 
10053  // If we find the pointer in the mesh data return true
10054  if (Mesh_pt->does_pointer_correspond_to_mesh_data(parameter_pt))
10055  {
10056  return true;
10057  }
10058 
10059  // Loop over the submeshes to handle the case of spine data
10060  const unsigned n_sub_mesh = this->nsub_mesh();
10061  // If there is only one mesh
10062  if (n_sub_mesh == 0)
10063  {
10064  if (SpineMesh* const spine_mesh_pt = dynamic_cast<SpineMesh*>(Mesh_pt))
10065  {
10066  if (spine_mesh_pt->does_pointer_correspond_to_spine_data(parameter_pt))
10067  {
10068  return true;
10069  }
10070  }
10071  }
10072  // Otherwise loop over the sub meshes
10073  else
10074  {
10075  // Assign global equation numbers first
10076  for (unsigned i = 0; i < n_sub_mesh; i++)
10077  {
10078  if (SpineMesh* const spine_mesh_pt =
10079  dynamic_cast<SpineMesh*>(Sub_mesh_pt[i]))
10080  {
10081  if (spine_mesh_pt->does_pointer_correspond_to_spine_data(
10082  parameter_pt))
10083  {
10084  return true;
10085  }
10086  }
10087  }
10088  }
10089 
10090  // If we have got here then the data is not stored in the problem, so return
10091  // false
10092  return false;
10093  }
10094 
10095 
10096  //=======================================================================
10097  /// A private helper function to
10098  /// calculate the derivatives with respect to the arc-length
10099  /// required for continuation. The arguments is the solution of the
10100  /// linear system,
10101  /// Jz = dR/dparameter, that gives du/dparameter and the direction
10102  /// output from the newton_solve_continuation function. The derivatives
10103  /// are stored in the ContinuationParameters namespace.
10104  //===================================================================
10106  {
10107  // Find the number of degrees of freedom in the problem
10108  // unsigned long n_dofs = ndof();
10109  // Find the number of local dofs in the problem
10110  const unsigned long ndof_local = Dof_distribution_pt->nrow_local();
10111 
10112  // Work out the continuation direction
10113  // The idea is that (du/ds)_{old} . (du/ds)_{new} >= 0
10114  // if the direction is to remain the same.
10115  // du/ds_{new} = [dlambda/ds; du/ds] = [dlambda/ds ; - dlambda/ds z]
10116  // so (du/ds)_{new} . (du/ds)_{old}
10117  // = dlambda/ds [1 ; - z] . [ Parameter_derivative ; Dof_derivatives]
10118  // = dlambda/ds (Parameter_derivative - Dof_derivative . z)
10119 
10120  // Create a local copy of z that can be redistributed without breaking
10121  // the constness of z
10122  DoubleVector local_z(z);
10123 
10124  // Redistribute z so that it has the (natural) dof distribution
10126 
10127  // Calculate the local contribution to the Continuation direction
10128  Continuation_direction = 0.0;
10129  // Cache the pointer to z
10130  double* const local_z_pt = local_z.values_pt();
10131  for (unsigned long l = 0; l < ndof_local; l++)
10132  {
10133  Continuation_direction -= dof_derivative(l) * local_z_pt[l];
10134  }
10135 
10136  // Now reduce if we have been distributed
10137 #ifdef OOMPH_HAS_MPI
10138  double cont_dir2 = Continuation_direction;
10139  if ((Dof_distribution_pt->distributed()) &&
10140  (Dof_distribution_pt->communicator_pt()->nproc() > 1))
10141  {
10142  MPI_Allreduce(&Continuation_direction,
10143  &cont_dir2,
10144  1,
10145  MPI_DOUBLE,
10146  MPI_SUM,
10147  Dof_distribution_pt->communicator_pt()->mpi_comm());
10148  }
10149  Continuation_direction = cont_dir2;
10150 #endif
10151 
10152  // Add parameter derivative
10154 
10155  // Calculate the magnitude of the du/ds Vector
10156 
10157  // Note that actually, we are usually approximating by using the value at
10158  // newton step just before convergence, which saves one additional
10159  // Newton solve.
10160 
10161  // First calculate the magnitude of du/dparameter, chi
10162  double chi = local_z.dot(local_z);
10163 
10164  // Calculate the current derivative of the parameter wrt the arc-length
10165  Parameter_derivative = 1.0 / sqrt(1.0 + Theta_squared * chi);
10166 
10167  // If the dot product of the current derivative wrt the Direction
10168  // is less than zero, switch the sign of the derivative to ensure
10169  // smooth continuation
10171  {
10172  Parameter_derivative *= -1.0;
10173  }
10174 
10175  // Resize the derivatives array, if necessary
10177  {
10178  if (Dof_derivative.size() != ndof_local)
10179  {
10180  Dof_derivative.resize(ndof_local, 0.0);
10181  }
10182  }
10183  // Calculate the new derivatives wrt the arc-length
10184  for (unsigned long l = 0; l < ndof_local; l++)
10185  {
10186  // This comes from the formulation J u_dot + dr/dlambda lambda_dot = 0
10187  // on the curve and then it follows that.
10188  dof_derivative(l) = -Parameter_derivative * local_z_pt[l];
10189  }
10190  }
10191 
10192  //=======================================================================
10193  /// A private helper function to
10194  /// calculate the derivatives with respect to the arc-length
10195  /// required for continuation using finite differences.
10196  //===================================================================
10198  double* const& parameter_pt)
10199  {
10200  // Find the number of values
10201  // const unsigned long n_dofs = this->ndof();
10202  // Find the number of local dofs in the problem
10203  const unsigned long ndof_local = Dof_distribution_pt->nrow_local();
10204 
10205  // Temporary storage for the finite-difference approximation to the helper
10206  Vector<double> z(ndof_local);
10207  double length = 0.0;
10208  // Calculate the change in values and contribution to total length
10209  for (unsigned long l = 0; l < ndof_local; l++)
10210  {
10211  z[l] = (*Dof_pt[l] - Dof_current[l]) / Ds_current;
10212  length += Theta_squared * z[l] * z[l];
10213  }
10214 
10215  // Reduce if parallel
10216 #ifdef OOMPH_HAS_MPI
10217  double length2 = length;
10218  if ((Dof_distribution_pt->distributed()) &&
10219  (Dof_distribution_pt->communicator_pt()->nproc() > 1))
10220  {
10221  MPI_Allreduce(&length,
10222  &length2,
10223  1,
10224  MPI_DOUBLE,
10225  MPI_SUM,
10226  Dof_distribution_pt->communicator_pt()->mpi_comm());
10227  }
10228  length = length2;
10229 #endif
10230 
10231  // Calculate change in parameter
10232  double Z = (*parameter_pt - Parameter_current) / Ds_current;
10233  length += Z * Z;
10234 
10235  // Scale the approximations to the derivatives
10236  length = sqrt(length);
10237  for (unsigned long l = 0; l < ndof_local; l++)
10238  {
10239  dof_derivative(l) = z[l] / length;
10240  }
10241  Parameter_derivative = Z / length;
10242  }
10243 
10244 
10245  /// Virtual function that is used to symmetrise the problem so that
10246  /// the current solution exactly satisfies any symmetries within the system.
10247  /// Used when adpativly solving pitchfork detection problems when small
10248  /// asymmetries in the coarse solution can be magnified
10249  /// leading to very inaccurate answers on the fine mesh.
10250  /// This is always problem-specific and must be filled in by the user
10251  /// The default issues a warning
10253  {
10254  std::ostringstream warn_message;
10255  warn_message
10256  << "Warning: This function is called after spatially adapting the\n"
10257  << "eigenfunction associated with a pitchfork bifurcation and should\n"
10258  << "ensure that the exact (anti-)symmetries of problem are enforced\n"
10259  << "within that eigenfunction. It is problem specific and must be\n"
10260  << "filled in by the user if required.\n"
10261  << "A sign of problems is if the slack paramter gets too large and\n"
10262  << "if the solution at the Pitchfork is not symmetric.\n";
10264  warn_message.str(),
10265  "Problem::symmetrise_eigenfunction_for_adaptive_pitchfork_tracking()",
10266  OOMPH_EXCEPTION_LOCATION);
10267  }
10268 
10269  //====================================================================
10270  /// Return pointer to the parameter that is used in the
10271  /// bifurcation detection. If we are not tracking a bifurcation then
10272  /// an error will be thrown by the AssemblyHandler
10273  //====================================================================
10275  {
10277  }
10278 
10279  //====================================================================
10280  /// Return the eigenfunction calculated as part of a
10281  /// bifurcation tracking process. If we are not tracking a bifurcation
10282  /// then an error will be thrown by the AssemblyHandler
10283  //======================================================================
10285  Vector<DoubleVector>& eigenfunction)
10286  {
10287  // Simply call the appropriate assembly handler function
10288  Assembly_handler_pt->get_eigenfunction(eigenfunction);
10289  }
10290 
10291  //============================================================
10292  /// Activate the fold tracking system by changing the assembly
10293  /// handler and initialising it using the parameter addressed
10294  /// by parameter_pt.
10295  //============================================================
10296  void Problem::activate_fold_tracking(double* const& parameter_pt,
10297  const bool& block_solve)
10298  {
10299  // Reset the assembly handler to default
10301  // Set the new assembly handler. Note that the constructor actually
10302  // solves the original problem to get some initial conditions, but
10303  // this is OK because the RHS is always evaluated before assignment.
10304  Assembly_handler_pt = new FoldHandler(this, parameter_pt);
10305 
10306  // If we are using a block solver, we must set the linear solver pointer
10307  // to the block fold solver. The present linear solver is
10308  // used by the block solver and so must be passed as an argument.
10309  // The destructor of the Fold handler returns the linear
10310  // solver to the original non-block version.
10311  if (block_solve)
10312  {
10314  }
10315  }
10316 
10317  //===============================================================
10318  /// Activate the generic bifurcation ///tracking system by changing the
10319  /// assembly handler and initialising it using the parameter addressed by
10320  /// parameter_pt.
10321  //============================================================
10322  void Problem::activate_bifurcation_tracking(double* const& parameter_pt,
10323  const DoubleVector& eigenvector,
10324  const bool& block_solve)
10325  {
10326  // Reset the assembly handler to default
10328  // Set the new assembly handler. Note that the constructor actually
10329  // solves the original problem to get some initial conditions, but
10330  // this is OK because the RHS is always evaluated before assignment.
10331  Assembly_handler_pt = new FoldHandler(this, parameter_pt, eigenvector);
10332 
10333  // If we are using a block solver, we must set the linear solver pointer
10334  // to the block fold solver. The present linear solver is
10335  // used by the block solver and so must be passed as an argument.
10336  // The destructor of the Fold handler returns the linear
10337  // solver to the original non-block version.
10338  if (block_solve)
10339  {
10341  }
10342  }
10343 
10344 
10345  //===============================================================
10346  /// Activate the generic bifurcation ///tracking system by changing the
10347  /// assembly handler and initialising it using the parameter addressed by
10348  /// parameter_pt.
10349  //============================================================
10350  void Problem::activate_bifurcation_tracking(double* const& parameter_pt,
10351  const DoubleVector& eigenvector,
10352  const DoubleVector& normalisation,
10353  const bool& block_solve)
10354  {
10355  // Reset the assembly handler to default
10357  // Set the new assembly handler. Note that the constructor actually
10358  // solves the original problem to get some initial conditions, but
10359  // this is OK because the RHS is always evaluated before assignment.
10361  new FoldHandler(this, parameter_pt, eigenvector, normalisation);
10362 
10363  // If we are using a block solver, we must set the linear solver pointer
10364  // to the block fold solver. The present linear solver is
10365  // used by the block solver and so must be passed as an argument.
10366  // The destructor of the Fold handler returns the linear
10367  // solver to the original non-block version.
10368  if (block_solve)
10369  {
10371  }
10372  }
10373 
10374 
10375  //==================================================================
10376  /// Activate the pitchfork tracking system by changing the assembly
10377  /// handler and initialising it using the parameter addressed
10378  /// by parameter_pt and a symmetry vector. The boolean flag is
10379  /// used to specify whether a block solver is used, default is true.
10380  //===================================================================
10381  void Problem::activate_pitchfork_tracking(double* const& parameter_pt,
10382  const DoubleVector& symmetry_vector,
10383  const bool& block_solve)
10384  {
10385  // Reset the assembly handler to default
10387 
10388  // Set the new assembly handler. Note that the constructor actually
10389  // solves the original problem to get some initial conditions, but
10390  // this is OK because the RHS is always evaluated before assignment.
10392  this, this->assembly_handler_pt(), parameter_pt, symmetry_vector);
10393 
10394  // If we are using a block solver, we must set the linear solver pointer
10395  // to the block pitchfork solver. The present linear solver is
10396  // used by the block solver and so must be passed as an argument.
10397  // The destructor of the PitchFork handler returns the linear
10398  // solver to the original non-block version.
10399  if (block_solve)
10400  {
10402  }
10403  }
10404 
10405 
10406  //============================================================
10407  /// Activate the hopf tracking system by changing the assembly
10408  /// handler and initialising it using the parameter addressed
10409  /// by parameter_pt.
10410  //============================================================
10411  void Problem::activate_hopf_tracking(double* const& parameter_pt,
10412  const bool& block_solve)
10413  {
10414  // Reset the assembly handler to default
10416  // Set the new assembly handler. Note that the constructor actually
10417  // solves the original problem to get some initial conditions, but
10418  // this is OK because the RHS is always evaluated before assignment.
10419  Assembly_handler_pt = new HopfHandler(this, parameter_pt);
10420 
10421  // If we are using a block solver, we must set the linear solver pointer
10422  // to the block hopf solver. The present linear solver is
10423  // used by the block solver and so must be passed as an argument.
10424  // The destructor of the Hopf handler returns the linear
10425  // solver to the original non-block version.
10426  if (block_solve)
10427  {
10429  }
10430  }
10431 
10432 
10433  //============================================================
10434  /// Activate the hopf tracking system by changing the assembly
10435  /// handler and initialising it using the parameter addressed
10436  /// by parameter_pt and the frequency and null vectors
10437  /// specified.
10438  //============================================================
10439  void Problem::activate_hopf_tracking(double* const& parameter_pt,
10440  const double& omega,
10441  const DoubleVector& null_real,
10442  const DoubleVector& null_imag,
10443  const bool& block_solve)
10444  {
10445  // Reset the assembly handler to default
10447  // Set the new assembly handler. Note that the constructor actually
10448  // solves the original problem to get some initial conditions, but
10449  // this is OK because the RHS is always evaluated before assignment.
10451  new HopfHandler(this, parameter_pt, omega, null_real, null_imag);
10452 
10453  // If we are using a block solver, we must set the linear solver pointer
10454  // to the block hopf solver. The present linear solver is
10455  // used by the block solver and so must be passed as an argument.
10456  // The destructor of the Hopf handler returns the linear
10457  // solver to the original non-block version.
10458  if (block_solve)
10459  {
10461  }
10462  }
10463 
10464 
10465  //===============================================================
10466  /// Reset the assembly handler to default
10467  //===============================================================
10469  {
10470  // If we have a non-default handler
10472  {
10473  // Delete the current assembly handler
10474  delete Assembly_handler_pt;
10475  // Reset the assembly handler
10477  }
10478  }
10479 
10480  //===================================================================
10481  /// This function takes one step of length ds in pseudo-arclength.The
10482  /// argument parameter_pt is a pointer to the parameter (global variable)
10483  /// that is being traded for arc-length. The function returns the next desired
10484  /// arc-length according to criteria based upon the desired number of Newton
10485  /// Iterations per solve.
10486  //=====================================================================
10487  double Problem::arc_length_step_solve(double* const& parameter_pt,
10488  const double& ds,
10489  const unsigned& max_adapt)
10490  {
10491  // First check that we shouldn't use the other interface
10492  // by checking that the parameter isn't already stored as data
10493  if (does_pointer_correspond_to_problem_data(parameter_pt))
10494  {
10495  std::ostringstream error_message;
10496  error_message
10497  << "The parameter addressed by " << parameter_pt << " with the value "
10498  << *parameter_pt
10499  << "\n is supposed to be used for arc-length contiunation,\n"
10500  << " but it is stored in a Data object used by the problem.\n\n"
10501  << "This is bad for two reasons:\n"
10502  << "1. If it's a variable in the problem, it must already have an\n"
10503  "associated equation, so it can't be used for continuation;\n"
10504  << "2. The problem data will be reorganised in memory during "
10505  "continuation,\n"
10506  << " which means that the pointer will become invalid.\n\n"
10507  << "If you are sure that this is what you want to do you must:\n"
10508  << "A. Ensure that the value is pinned (don't worry we'll shout again "
10509  "if not)\n"
10510  << "B. Use the alternative interface\n"
10511  << " Problem::arc_length_step_solve(Data*,unsigned,...)\n"
10512  << " which uses a pointer to the data object and not the raw double "
10513  "pointer."
10514  << std::endl;
10515  throw OomphLibError(
10516  error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
10517  }
10518 
10519 
10520  // If we are using the continuation timestepper
10522  {
10523  // Has the timestepper already been added to the problem
10524  bool continuation_time_stepper_added = false;
10525  const unsigned n_time_steppers = this->ntime_stepper();
10526  for (unsigned i = 0; i < n_time_steppers; i++)
10527  {
10529  {
10530  continuation_time_stepper_added = true;
10531  break;
10532  }
10533  }
10534 
10535  // If not add it
10536  if (!continuation_time_stepper_added)
10537  {
10538  oomph_info << "Adding the continuation time stepper\n";
10540  }
10541 
10542  // Need to treat case of eigenproblems and bifurcation detection/tracking
10543  // here
10544 
10545  // Backup the current timesteppers for each mesh!
10546 
10547 
10548  // If an arc length step has not been taken then set the timestepper
10549  if (!Arc_length_step_taken)
10550  {
10551  // Set the continuation timestepper for all data in the problem
10554  << " equation numbers allocated for continuation\n";
10555  }
10556 
10557  } // End of continuation time stepper case
10558 
10559 
10560  // Just call the helper function (parameter is not from data)
10561  return arc_length_step_solve_helper(parameter_pt, ds, max_adapt);
10562  }
10563 
10564 
10565  //===================================================================
10566  /// This function takes one step of length ds in pseudo-arclength.The
10567  /// argument data_pt is a pointer to the data that holds the
10568  /// parameter (global variable)
10569  /// that is being traded for arc-length. The exact value is located at
10570  /// the location given by data_index.
10571  /// The function returns the next desired
10572  /// arc-length according to criteria based upon the desired number of Newton
10573  /// Iterations per solve.
10574  //=====================================================================
10575  double Problem::arc_length_step_solve(Data* const& data_pt,
10576  const unsigned& data_index,
10577  const double& ds,
10578  const unsigned& max_adapt)
10579  {
10580  // Firstly check that the data is pinned
10581  if (!data_pt->is_pinned(data_index))
10582  {
10583  std::ostringstream error_stream;
10584  error_stream << "The value at index " << data_index
10585  << " in the data object to be used for continuation\n"
10586  << "is not pinned, which means that it is already a\n"
10587  << "variable in the problem "
10588  << "and cannot be used for continuation.\n\n"
10589  << "Please correct your formulation by either:\n"
10590  << "A. Pinning the value"
10591  << "\n or \n"
10592  << "B. Using a different parameter for continuation"
10593  << std::endl;
10594  throw OomphLibError(
10595  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
10596  }
10597 
10598 
10599  // If we are using the continuation timestepper
10601  {
10602  // Has the timestepper already been added to the problem
10603  bool continuation_time_stepper_added = false;
10604  const unsigned n_time_steppers = this->ntime_stepper();
10605  for (unsigned i = 0; i < n_time_steppers; i++)
10606  {
10608  {
10609  continuation_time_stepper_added = true;
10610  break;
10611  }
10612  }
10613 
10614  // If not add it
10615  if (!continuation_time_stepper_added)
10616  {
10617  oomph_info << "Adding the continuation time stepper\n";
10619  }
10620 
10621  // Need to treat case of eigenproblems and bifurcation detection/tracking
10622  // here
10623 
10624 
10625  // Backup the current timesteppers for each mesh!
10626 
10627 
10628  // If an arc length step has not been taken then set the timestepper
10629  if (!Arc_length_step_taken)
10630  {
10631  // Set the continuation timestepper for all data in the problem
10634  << " equation numbers allocated for continuation\n";
10635  }
10636 
10637 
10638  } // End of continuation time stepper case
10639 
10640 
10641  // Now make a pointer to the (newly allocated) data object
10642  double* parameter_pt = data_pt->value_pt(data_index);
10643  // Call the helper function, this will change the parameter_pt if
10644  // the data storage is changed (if the timestepper has to be changed,
10645  // which happens if this is the first time that a continuation step is
10646  // taken)
10647  // ALH: Don't think this is true because it has happened above....
10648  return arc_length_step_solve_helper(parameter_pt, ds, max_adapt);
10649  }
10650 
10651  //======================================================================
10652  /// Private helper function that is used to set the appropriate
10653  /// pinned values for continuation. If the data is pinned, the its
10654  /// current value is always the same as the original value and
10655  /// the derivative is always zero. If these are not set properly
10656  /// then interpolation and projection in spatial adaptivity will
10657  /// not give the best answers.
10658  //=====================================================================
10660  {
10661  // Set the consistent values for the global mesh
10664 
10665  // Deal with the spine meshes additional numbering separately
10666  const unsigned n_sub_mesh = this->nsub_mesh();
10667  // If there is only one mesh
10668  if (n_sub_mesh == 0)
10669  {
10670  if (SpineMesh* const spine_mesh_pt = dynamic_cast<SpineMesh*>(Mesh_pt))
10671  {
10672  spine_mesh_pt->set_consistent_pinned_spine_values_for_continuation(
10674  }
10675  // If it's a triangle mesh the we need to set the
10676  }
10677  // Otherwise loop over the sub meshes
10678  else
10679  {
10680  // Assign global equation numbers first
10681  for (unsigned i = 0; i < n_sub_mesh; i++)
10682  {
10683  if (SpineMesh* const spine_mesh_pt =
10684  dynamic_cast<SpineMesh*>(Sub_mesh_pt[i]))
10685  {
10686  spine_mesh_pt->set_consistent_pinned_spine_values_for_continuation(
10688  }
10689  }
10690  }
10691 
10692  // Also set time stepper for global data
10693  const unsigned n_global = Global_data_pt.size();
10694  for (unsigned i = 0; i < n_global; ++i)
10695  {
10697  }
10698  }
10699 
10700 
10701  //===================================================================
10702  /// This function takes one step of length ds in pseudo-arclength.The
10703  /// argument parameter_pt is a pointer to the parameter (global variable)
10704  /// that is being traded for arc-length. The function returns the next desired
10705  /// arc-length according to criteria based upon the desired number of Newton
10706  /// Iterations per solve.
10707  //=====================================================================
10708  double Problem::arc_length_step_solve_helper(double* const& parameter_pt,
10709  const double& ds,
10710  const unsigned& max_adapt)
10711  {
10712  //----------------------MAKE THE PROBLEM STEADY-----------------------
10713  // Loop over the timesteppers and make them (temporarily) steady.
10714  // We can only do continuation for steady problems!
10715  unsigned n_time_steppers = ntime_stepper();
10716  // Vector of bools to store the is_steady status of the various
10717  // timesteppers when we came in here
10718  std::vector<bool> was_steady(n_time_steppers);
10719 
10720  // Loop over them all and make them (temporarily) static
10721  for (unsigned i = 0; i < n_time_steppers; i++)
10722  {
10723  was_steady[i] = time_stepper_pt(i)->is_steady();
10725  }
10726 
10727 
10728  // Max number of solves
10729  unsigned max_solve = max_adapt + 1;
10730  // Storage for newton steps in each adaptation
10731  unsigned max_count_in_adapt_loop = 0;
10732 
10733 
10734  //----SET UP MEMORY FOR QUANTITIES THAT ARE REQUIRED OUTSIDE THE LOOP----
10735 
10736  // Assign memory for solutions of the equations Jz = du/dparameter
10737  // This is needed here (outside the loop), so that we can save on
10738  // one linear solve when calculating the derivatives wrt the arc-length
10739  DoubleVector z;
10740 
10741 
10742  // Store sign of the Jacobian, used for bifurcation detection
10743  // If this is the first time that we are calling the arc-length solver,
10744  // this should not be used.
10745  int previous_sign = Sign_of_jacobian;
10746 
10747  // Flag to indicate a sign change
10748  bool SIGN_CHANGE = false;
10749 
10750 
10751  // Adaptation loop
10752  for (unsigned isolve = 0; isolve < max_solve; ++isolve)
10753  {
10754  // Only adapt after the first solve has been done
10755  if (isolve > 0)
10756  {
10757  unsigned n_refined;
10758  unsigned n_unrefined;
10759 
10760  // Adapt problem
10761  adapt(n_refined, n_unrefined);
10762 
10763 #ifdef OOMPH_HAS_MPI
10764  // Adaptation only converges if ALL the processes have no
10765  // refinement or unrefinement to perform
10766  unsigned total_refined = 0;
10767  unsigned total_unrefined = 0;
10769  {
10770  MPI_Allreduce(&n_refined,
10771  &total_refined,
10772  1,
10773  MPI_UNSIGNED,
10774  MPI_SUM,
10775  this->communicator_pt()->mpi_comm());
10776  n_refined = total_refined;
10777  MPI_Allreduce(&n_unrefined,
10778  &total_unrefined,
10779  1,
10780  MPI_UNSIGNED,
10781  MPI_SUM,
10782  this->communicator_pt()->mpi_comm());
10783  n_unrefined = total_unrefined;
10784  }
10785 #endif
10786 
10787  oomph_info << "---> " << n_refined << " elements were refined, and "
10788  << n_unrefined << " were unrefined"
10789 #ifdef OOMPH_HAS_MPI
10790  << ", in total (over all processors).\n";
10791 #else
10792  << ".\n";
10793 #endif
10794 
10795 
10796  // Check convergence of adaptation cycle
10797  if ((n_refined == 0) && (n_unrefined == 0))
10798  {
10799  oomph_info << "\n \n Solution is fully converged in "
10800  << "Problem::newton_solver(). \n \n ";
10801  break;
10802  }
10803  }
10804 
10805  //----------SAVE THE INITIAL VALUES, IN CASE THE STEP FAILS-----------
10806 
10807  // Find the number of local dofs
10808  unsigned ndof_local = Dof_distribution_pt->nrow_local();
10809 
10810  // Only need to do this in the first loop
10811  if (isolve == 0)
10812  {
10814  {
10815  // Safety check, set up the array of dof derivatives, if necessary
10816  // The distribution is the same as the (natural) distribution of the
10817  // dofs
10818  if (Dof_derivative.size() != ndof_local)
10819  {
10820  Dof_derivative.resize(ndof_local, 0.0);
10821  }
10822 
10823  // Safety check, set up the array of curren values, if necessary
10824  // Again the distribution reflects the (natural) distribution of the
10825  // dofs
10826  if (Dof_current.size() != ndof_local)
10827  {
10828  Dof_current.resize(ndof_local);
10829  }
10830  }
10831 
10832  // Save the current value of the parameter
10833  Parameter_current = *parameter_pt;
10834 
10835  // Save the current values of the degrees of freedom
10836  for (unsigned long l = 0; l < ndof_local; l++)
10837  {
10838  dof_current(l) = *Dof_pt[l];
10839  }
10840 
10841  // Set the value of ds_current
10842  Ds_current = ds;
10843  }
10844 
10845  // Counter for the number of newton steps
10846  unsigned count = 0;
10847 
10848  // Flag to indicate a successful step
10849  bool STEP_REJECTED = false;
10850 
10851 
10852  // Set the appropriate initial conditions for the pinned data
10854  {
10856  }
10857 
10858  // Loop around the step in arc-length
10859  do
10860  {
10861  // Check that the step has not fallen below the minimum tolerance
10862  if (std::fabs(Ds_current) < Minimum_ds)
10863  {
10864  std::ostringstream error_message;
10865  error_message << "DESIRED ARC-LENGTH STEP " << Ds_current
10866  << " HAS FALLEN BELOW MINIMUM TOLERANCE, " << Minimum_ds
10867  << std::endl;
10868 
10869  throw OomphLibError(error_message.str(),
10870  OOMPH_CURRENT_FUNCTION,
10871  OOMPH_EXCEPTION_LOCATION);
10872  }
10873 
10874  // Assume that we shall accept the step
10875  STEP_REJECTED = false;
10876 
10877  // Set initial value of the parameter
10879 
10880  // Perform any actions...
10881  actions_after_parameter_increase(parameter_pt);
10882 
10883  Ds_current = (*parameter_pt - Parameter_current) / Parameter_derivative;
10884 
10885  // Loop over the (local) variables and set their initial values
10886  for (unsigned long l = 0; l < ndof_local; l++)
10887  {
10888  *Dof_pt[l] = dof_current(l) + dof_derivative(l) * Ds_current;
10889  }
10890 
10891  // Actually do the newton solve stage for the continuation problem
10892  try
10893  {
10894  count = newton_solve_continuation(parameter_pt, z);
10895  }
10896  // Catch any exceptions thrown in the Newton solver
10897  catch (NewtonSolverError& error)
10898  {
10899  // Check whether it's the linear solver
10900  if (error.linear_solver_error)
10901  {
10902  std::ostringstream error_stream;
10903  error_stream << std::endl
10904  << "USER-DEFINED ERROR IN NEWTON SOLVER " << std::endl;
10905  oomph_info << "ERROR IN THE LINEAR SOLVER" << std::endl;
10906  throw OomphLibError(error_stream.str(),
10907  OOMPH_CURRENT_FUNCTION,
10908  OOMPH_EXCEPTION_LOCATION);
10909  }
10910  // Otherwise mark the step as having failed
10911  else
10912  {
10913  oomph_info << "STEP REJECTED --- TRYING AGAIN" << std::endl;
10914  STEP_REJECTED = true;
10915  // Let's take a smaller step
10916  Ds_current *= (2.0 / 3.0);
10917  }
10918  }
10919  } while (STEP_REJECTED); // continue until a step is accepted
10920 
10921  // Set the maximum count
10922  if (count > max_count_in_adapt_loop)
10923  {
10924  max_count_in_adapt_loop = count;
10925  }
10926  } /// end of adaptation loop
10927 
10928  // Only recalculate the derivatives if there has been a Newton solve
10929  // If not, the previous values should be close enough
10930  if (max_count_in_adapt_loop > 0)
10931  {
10932  //--------------------CHECK FOR POTENTIAL BIFURCATIONS-------------
10934  {
10935  // If the sign of the jacobian is zero issue a warning
10936  if (Sign_of_jacobian == 0)
10937  {
10938  std::string error_message =
10939  "The sign of the jacobian is zero after a linear solve\n";
10940  error_message += "Either the matrix is singular (unlikely),\n";
10941  error_message += "or the linear solver cannot compute the "
10942  "determinant of the matrix;\n";
10943  error_message += "e.g. an iterative linear solver.\n";
10944  error_message +=
10945  "If the latter, bifurcation detection must be via an eigensolver\n";
10946  OomphLibWarning(error_message,
10947  "Problem::arc_length_step_solve",
10948  OOMPH_EXCEPTION_LOCATION);
10949  }
10950  // If this is the first step, we cannot rely on the previous value
10951  // of the jacobian so set the previous sign to the present sign
10952  if (!Arc_length_step_taken)
10953  {
10954  previous_sign = Sign_of_jacobian;
10955  }
10956  // If we have detected a sign change in the last converged Jacobian,
10957  // it must be a turning point or bifurcation
10958  if (Sign_of_jacobian != previous_sign)
10959  {
10960  // There has been, at least, one sign change
10962 
10963  // The sign has changed this time
10964  SIGN_CHANGE = true;
10965 
10966  // Calculate the dot product of the approximate null vector
10967  // of the Jacobian matrix ((badly) approximated by z)
10968  // and the vectors of derivatives of the residuals wrt the
10969  // global parameter
10970  // If this is small it is a bifurcation rather than a turning point.
10971  // Get the derivative wrt global parameter
10972  // DoubleVector dparam;
10973  // get_derivative_wrt_global_parameter(parameter_pt,dparam);
10974  // Calculate the dot product
10975  // double dot=0.0;
10976  // for(unsigned long n=0;n<n_dofs;++n) {dot += dparam[n]*z[n];}
10977  // z.dot(dparam);
10978 
10979  // Write the output message
10980  std::ostringstream message;
10981  message
10982  << "-----------------------------------------------------------";
10983  message << std::endl
10984  << "SIGN CHANGE IN DETERMINANT OF JACOBIAN: " << std::endl;
10985  message << "BIFURCATION OR TURNING POINT DETECTED BETWEEN "
10986  << Parameter_current << " AND " << *parameter_pt << std::endl;
10987  // message << "APPROXIMATE DOT PRODUCT : " << dot << "," << std::endl;
10988  // message << "IF CLOSE TO ZERO WE HAVE A BIFURCATION; ";
10989  // message << "OTHERWISE A TURNING POINT" << std::endl;
10990  message
10991  << "-----------------------------------------------------------"
10992  << std::endl;
10993 
10994  // Write the message to standard output
10995  oomph_info << message.str();
10996 
10997  // Open the information file for appending
10998  std::ofstream bifurcation_info("bifurcation_info",
10999  std::ios_base::app);
11000  // Write the message to the file
11001  bifurcation_info << message.str();
11002  bifurcation_info.close();
11003  }
11004  }
11005 
11006  // Calculate the derivatives required for the next stage of continuation
11007  // In this we pass the last value of z (i.e. approximation)
11009  {
11011  }
11012  // Or use finite differences
11013  else
11014  {
11016  }
11017 
11018  // If it's the first step then the value of the next step should
11019  // be the change in parameter divided by the parameter derivative
11020  // to obtain approximately the same parameter change
11021  if (!Arc_length_step_taken)
11022  {
11023  Ds_current = (*parameter_pt - Parameter_current) / Parameter_derivative;
11024  }
11025 
11026  // We have taken our first step
11027  Arc_length_step_taken = true;
11028  }
11029  // If there has not been a newton step then we still need to estimate
11030  // the derivatives in the arc length direction
11031  else
11032  {
11033  // Default is to calculate the continuation derivatives by solving the
11034  // linear system. We must do this to ensure that the derivatives are in
11035  // sync It could lead to problems near turning points when we should
11036  // really be solving an eigenproblem, but seems OK so far!
11037 
11038  // Save the current sign of the jacobian
11039  int temp_sign = Sign_of_jacobian;
11040 
11041  // Calculate the continuation derivatives, which includes a solve
11042  // of the linear system if not using finite differences
11044  {
11045  calculate_continuation_derivatives(parameter_pt);
11046  }
11047  // Otherwise use finite differences
11048  else
11049  {
11051  }
11052 
11053  // Reset the sign of the jacobian, just in case the sign has changed when
11054  // solving the continuation derivatives. The sign change will be picked
11055  // up on the next continuation step.
11056  Sign_of_jacobian = temp_sign;
11057  }
11058 
11059  // Reset the is_steady status of all timesteppers that
11060  // weren't already steady when we came in here and reset their
11061  // weights
11062  for (unsigned i = 0; i < n_time_steppers; i++)
11063  {
11064  if (!was_steady[i])
11065  {
11067  }
11068  }
11069 
11070  // If we are trying to find a bifurcation and the first sign change
11071  // has occured, use bisection
11074  {
11075  // If there has been a sign change we need to half the step size
11076  // and reverse the direction
11077  if (SIGN_CHANGE)
11078  {
11079  Ds_current *= -0.5;
11080  }
11081  // Otherwise
11082  else
11083  {
11084  // The size of the bracketed interval is always
11085  // 2ds - Ds_current (this will work even if the original step failed)
11086  // We want our new step size to be half this
11087  Ds_current = ds - 0.5 * Ds_current;
11088  }
11089  // Return the desired value of the step
11090  return Ds_current;
11091  }
11092 
11093  // If fewer than the desired number of Newton Iterations, increase the step
11094  if (max_count_in_adapt_loop < Desired_newton_iterations_ds)
11095  {
11096  return Ds_current * 1.5;
11097  }
11098  // If more than the desired number of Newton Iterations, reduce the step
11099  if (max_count_in_adapt_loop > Desired_newton_iterations_ds)
11100  {
11101  return Ds_current * (2.0 / 3.0);
11102  }
11103  // Otherwise return the step just taken
11104  return Ds_current;
11105  }
11106 
11107 
11108  //=======================================================================
11109  /// Take an explicit timestep of size dt
11110  //======================================================================
11111  void Problem::explicit_timestep(const double& dt, const bool& shift_values)
11112  {
11113 #ifdef PARANOID
11114  if (this->explicit_time_stepper_pt() == 0)
11115  {
11116  throw OomphLibError("Explicit time stepper pointer is null in problem.",
11117  OOMPH_EXCEPTION_LOCATION,
11118  OOMPH_CURRENT_FUNCTION);
11119  }
11120 #endif
11121 
11122  // Firstly we shift the time values
11123  if (shift_values)
11124  {
11126  }
11127  // Set the current value of dt, if we can
11128  if (time_pt()->ndt() > 0)
11129  {
11130  time_pt()->dt() = dt;
11131  }
11132 
11133  // Take the explicit step
11134  this->explicit_time_stepper_pt()->timestep(this, dt);
11135  }
11136 
11137 
11138  //========================================================================
11139  /// Do one timestep of size dt using Newton's method with the specified
11140  /// tolerance and linear solver defined as member data of the Problem class.
11141  /// This will be the most commonly used version
11142  /// of unsteady_newton_solve, in which the time values are always shifted
11143  /// This does not include any kind of adaptativity. If the solution fails to
11144  /// converge the program will end.
11145  //========================================================================
11146  void Problem::unsteady_newton_solve(const double& dt)
11147  {
11148  // We shift the values, so shift_values is true
11149  unsteady_newton_solve(dt, true);
11150  }
11151 
11152  //========================================================================
11153  /// Do one timestep forward of size dt using Newton's method with the
11154  /// specified tolerance and linear solver defined via member data of the
11155  /// Problem class.
11156  /// The boolean flag shift_values is used to control whether the time values
11157  /// should be shifted or not.
11158  //========================================================================
11159  void Problem::unsteady_newton_solve(const double& dt,
11160  const bool& shift_values)
11161  {
11162  // Shift the time values and the dts, according to the control flag
11163  if (shift_values)
11164  {
11166  }
11167 
11168  // Advance global time and set current value of dt
11169  time_pt()->time() += dt;
11170  time_pt()->dt() = dt;
11171 
11172  // Find out how many timesteppers there are
11173  unsigned n_time_steppers = ntime_stepper();
11174 
11175  // Loop over them all and set the weights
11176  for (unsigned i = 0; i < n_time_steppers; i++)
11177  {
11179  }
11180 
11181  // Run the individual timesteppers actions before timestep. These need to
11182  // be before the problem's actions_before_implicit_timestep so that the
11183  // boundary conditions are set consistently.
11184  for (unsigned i = 0; i < n_time_steppers; i++)
11185  {
11187  }
11188 
11189  // Now update anything that needs updating before the timestep
11190  // This could be time-dependent boundary conditions, for example.
11192 
11193  try
11194  {
11195  // Solve the non-linear problem for this timestep with Newton's method
11196  newton_solve();
11197  }
11198  // Catch any exceptions thrown in the Newton solver
11199  catch (NewtonSolverError& error)
11200  {
11201  oomph_info << std::endl
11202  << "USER-DEFINED ERROR IN NEWTON SOLVER " << std::endl;
11203  // Check whether it's the linear solver
11204  if (error.linear_solver_error)
11205  {
11206  oomph_info << "ERROR IN THE LINEAR SOLVER" << std::endl;
11207  }
11208  // Check to see whether we have reached Max_iterations
11209  else if (error.iterations == Max_newton_iterations)
11210  {
11211  oomph_info << "MAXIMUM NUMBER OF ITERATIONS (" << error.iterations
11212  << ") REACHED WITHOUT CONVERGENCE " << std::endl;
11213  }
11214  // If not, it must be that we have exceeded the maximum residuals
11215  else
11216  {
11217  oomph_info << "MAXIMUM RESIDUALS: " << error.maxres
11218  << " EXCEEDS PREDEFINED MAXIMUM " << Max_residuals
11219  << std::endl;
11220  }
11221  // Die horribly!!
11222  std::ostringstream error_stream;
11223  error_stream << "Error occured in unsteady Newton solver. " << std::endl;
11224  throw OomphLibError(
11225  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
11226  }
11227 
11228  // Run the individual timesteppers actions, these need to be before the
11229  // problem's actions_after_implicit_timestep so that the time step is
11230  // finished before the problem does any auxiliary calculations (e.g. in
11231  // semi-implicit micromagnetics the calculation of magnetostatic field).
11232  for (unsigned i = 0; i < n_time_steppers; i++)
11233  {
11235  }
11236 
11237 
11238  // Now update anything that needs updating after the timestep
11241  }
11242 
11243  //=======================================================================
11244  /// Attempt to take one timestep forward using dt_desired. The error control
11245  /// parameter, epsilon, is used to specify the desired approximate value of
11246  /// the global error norm per timestep. The routine returns the value an
11247  /// estimate of the next value of dt that should be taken.
11248  //=======================================================================
11249  double Problem::adaptive_unsteady_newton_solve(const double& dt_desired,
11250  const double& epsilon)
11251  {
11252  // We always want to shift the time values
11253  return adaptive_unsteady_newton_solve(dt_desired, epsilon, true);
11254  }
11255 
11256 
11257  //=======================================================================
11258  /// Attempt to take one timestep forward using the dt_desired.
11259  /// This is the driver for a number of adaptive solvers. If the solution
11260  /// fails to converge at a given timestep, the routine will automatically
11261  /// halve the time step and try again, until the time step falls below the
11262  /// specified minimum value. The routine returns the value an estimate
11263  /// of the next value of dt that should be taken.
11264  /// Timestep is also rejected if the error estimate post-solve
11265  /// (computed by global_temporal_error_norm()) exceeds epsilon.
11266  /// This behaviour can be over-ruled by setting the protected
11267  /// boolean Problem::Keep_temporal_error_below_tolerance to false.
11268  //========================================================================
11269  double Problem::adaptive_unsteady_newton_solve(const double& dt_desired,
11270  const double& epsilon,
11271  const bool& shift_values)
11272  {
11273  // First, we need to backup the existing dofs, in case the timestep is
11274  // rejected
11275 
11276  // Find total number of dofs on current processor
11277  unsigned n_dof_local = dof_distribution_pt()->nrow_local();
11278 
11279  // Now set up a Vector to hold current values
11280  Vector<double> dofs_current(n_dof_local);
11281 
11282  // Load values into dofs_current
11283  for (unsigned i = 0; i < n_dof_local; i++) dofs_current[i] = dof(i);
11284 
11285  // Store the time
11286  double time_current = time_pt()->time();
11287 
11288  // Flag to detect whether the timestep has been rejected or not
11289  bool reject_timestep = 0;
11290 
11291  // Flag to detect whether any of the timesteppers are adaptive
11292  unsigned adaptive_flag = 0;
11293 
11294  // The value of the actual timestep, by default the same as desired timestep
11295  double dt_actual = dt_desired;
11296 
11297  // Find out whether any of the timesteppers are adaptive
11298  unsigned n_time_steppers = ntime_stepper();
11299  for (unsigned i = 0; i < n_time_steppers; i++)
11300  {
11301  if (time_stepper_pt(i)->adaptive_flag())
11302  {
11303  adaptive_flag = 1;
11304  break;
11305  }
11306  }
11307 
11308  // Shift the time_values according to the control flag
11309  if (shift_values)
11310  {
11312  }
11313 
11314  // This loop surrounds the adaptive time-stepping and will not be broken
11315  // until a timestep is accepted
11316  do
11317  {
11318  // Initially we assume that this step will succeed and that this dt
11319  // value is ok.
11320  reject_timestep = 0;
11321  double dt_rescaling_factor = 1.0;
11322 
11323  // Set the new time and value of dt
11324  time_pt()->time() += dt_actual;
11325  time_pt()->dt() = dt_actual;
11326 
11327  // Loop over all timesteppers and set the weights and predictor weights
11328  for (unsigned i = 0; i < n_time_steppers; i++)
11329  {
11330  // If the time_stepper is non-adaptive, this will be zero
11333  }
11334 
11335  // Now calculate the predicted values for the all data and all positions
11337 
11338  // Run the individual timesteppers actions before timestep. These need to
11339  // be before the problem's actions_before_implicit_timestep so that the
11340  // boundary conditions are set consistently.
11341  for (unsigned i = 0; i < n_time_steppers; i++)
11342  {
11344  }
11345 
11346  // Do any updates/boundary conditions changes here
11348 
11349  // Attempt to solve the non-linear system
11350  try
11351  {
11352  // Solve the non-linear problem at this timestep
11353  newton_solve();
11354  }
11355  // Catch any exceptions thrown
11356  catch (NewtonSolverError& error)
11357  {
11358  // If it's a solver error then die
11359  if (error.linear_solver_error ||
11361  {
11362  std::string error_message = "USER-DEFINED ERROR IN NEWTON SOLVER\n";
11363  error_message += "ERROR IN THE LINEAR SOLVER\n";
11364 
11365  // Die
11366  throw OomphLibError(
11367  error_message, OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
11368  }
11369  else
11370  {
11371  // Reject the timestep, if we have an exception
11372  oomph_info << "TIMESTEP REJECTED" << std::endl;
11373  reject_timestep = 1;
11374 
11375  // Half the time step
11376  dt_rescaling_factor = Timestep_reduction_factor_after_nonconvergence;
11377  }
11378  }
11379 
11380  // Run the individual timesteppers actions, these need to be before the
11381  // problem's actions_after_implicit_timestep so that the time step is
11382  // finished before the problem does any auxiliary calculations (e.g. in
11383  // semi-implicit micromagnetics the calculation of magnetostatic field).
11384  for (unsigned i = 0; i < n_time_steppers; i++)
11385  {
11387  }
11388 
11389  // Update anything that needs updating after the timestep
11391 
11392  // If we have an adapative timestepper (and we haven't already failed)
11393  // then calculate the error estimate and rescaling factor.
11394  if (adaptive_flag && !reject_timestep)
11395  {
11396  // Once timestep has been accepted can do fancy error processing
11397  // Set the error weights
11398  for (unsigned i = 0; i < n_time_steppers; i++)
11399  {
11401  }
11402 
11403  // Get a global error norm to use in adaptivity (as specified by the
11404  // problem sub-class writer). Prevent a divide by zero if the solution
11405  // gives very close to zero error. Error norm should never be negative
11406  // but use absolute value just in case.
11407  double error = std::max(std::abs(global_temporal_error_norm()), 1e-12);
11408 
11409  // Calculate the scaling factor
11410  dt_rescaling_factor = std::pow(
11411  (epsilon / error), (1.0 / (1.0 + time_stepper_pt()->order())));
11412 
11413  oomph_info << "Timestep scaling factor is " << dt_rescaling_factor
11414  << std::endl;
11415  oomph_info << "Estimated timestepping error is " << error << std::endl;
11416 
11417 
11418  // Do we have to do it again?
11419  if (error > epsilon)
11420  {
11421  oomph_info << "Estimated timestepping error " << error
11422  << " exceeds tolerance " << epsilon << " ";
11424  {
11425  oomph_info << " --> rejecting timestep." << std::endl;
11426  reject_timestep = 1;
11427  }
11428  else
11429  {
11430  oomph_info << " ...but we're not rejecting the timestep"
11431  << std::endl;
11432  }
11433  oomph_info
11434  << "Note: This behaviour can be adjusted by changing the protected "
11435  << "boolean" << std::endl
11436  << std::endl
11437  << " Problem::Keep_temporal_error_below_tolerance" << std::endl;
11438  }
11439 
11440 
11441  } // End of if adaptive flag
11442 
11443 
11444  // Calculate the next time step size and check it's ok
11445  // ============================================================
11446 
11447  // Calculate the possible next time step, if no error conditions
11448  // trigger.
11449  double new_dt_candidate = dt_rescaling_factor * dt_actual;
11450 
11451  // Check that the scaling factor is within the allowed range
11452  if (dt_rescaling_factor > DTSF_max_increase)
11453  {
11454  oomph_info << "Tried to increase dt by the ratio "
11455  << dt_rescaling_factor << " which is above the maximum ("
11457  << "). Attempting to increase by the maximum ratio instead."
11458  << std::endl;
11459  new_dt_candidate = DTSF_max_increase * dt_actual;
11460  }
11461  // If we have already rejected the timestep then don't do this check
11462  // because DTSF will definitely be too small.
11463  else if ((!reject_timestep) && (dt_rescaling_factor <= DTSF_min_decrease))
11464  {
11465  // Handle this special case where we want to continue anyway (usually
11466  // Minimum_dt_but_still_proceed = -1 so this has no effect).
11467  if (new_dt_candidate < Minimum_dt_but_still_proceed)
11468  {
11469  oomph_info
11470  << "Warning: Adaptation of timestep to ensure satisfaction\n"
11471  << " of error bounds during adaptive timestepping\n"
11472  << " would lower dt below \n"
11473  << " Problem::Minimum_dt_but_still_proceed="
11474  << Minimum_dt_but_still_proceed << "\n"
11475  << " ---> We're continuing with present timestep.\n"
11476  << std::endl;
11477  dt_rescaling_factor = 1.0;
11478  // ??ds shouldn't we set new_dt_candidate =
11479  // Minimum_dt_but_still_proceed here, rather than not changing dt at
11480  // all?
11481  }
11482  else
11483  {
11484  // Otherwise reject
11485  oomph_info << "Timestep would decrease by " << dt_rescaling_factor
11486  << " which is less than the minimum scaling factor "
11487  << DTSF_min_decrease << std::endl;
11488  oomph_info << "TIMESTEP REJECTED" << std::endl;
11489  reject_timestep = 1;
11490  }
11491  }
11492 
11493  // Now check that the new dt is within the allowed range
11494  if (new_dt_candidate > Maximum_dt)
11495  {
11496  oomph_info << "Tried to increase dt to " << new_dt_candidate
11497  << " which is above the maximum (" << Maximum_dt
11498  << "). I increased it to the maximum value instead.";
11499  dt_actual = Maximum_dt;
11500  }
11501  else if (new_dt_candidate < Minimum_dt)
11502  {
11503  std::ostringstream err;
11504  err << "Tried to reduce dt to " << new_dt_candidate
11505  << " which is less than the minimum dt (" << Minimum_dt << ")."
11506  << std::endl;
11507  throw OomphLibError(
11508  err.str(), OOMPH_EXCEPTION_LOCATION, OOMPH_CURRENT_FUNCTION);
11509  }
11510  else
11511  {
11512  dt_actual = new_dt_candidate;
11513  }
11514 
11515 
11517 
11518 
11519  // If we are rejecting this attempt then revert the dofs etc.
11520  if (reject_timestep)
11521  {
11522  // Reset the time
11523  time_pt()->time() = time_current;
11524 
11525  // Reload the dofs
11526  unsigned ni = dofs_current.size();
11527  for (unsigned i = 0; i < ni; i++)
11528  {
11529  dof(i) = dofs_current[i];
11530  }
11531 
11532 #ifdef OOMPH_HAS_MPI
11533  // Synchronise the solution on different processors (on each submesh)
11534  this->synchronise_all_dofs();
11535 #endif
11536 
11537  // Call all "after" actions, e.g. to handle mesh updates
11543  }
11544 
11545  }
11546  // Keep this loop going until we accept the timestep
11547  while (reject_timestep);
11548 
11549  // Once the timestep has been accepted, return the time step that should be
11550  // used next time.
11551  return dt_actual;
11552  }
11553 
11554 
11555  //=======================================================================
11556  /// Private helper function to perform
11557  /// unsteady "doubly" adaptive Newton solve: Does temporal
11558  /// adaptation first, i.e. we try to do a timestep with an increment
11559  /// of dt, and adjusting dt until the solution on the given mesh satisfies
11560  /// the temporal error measure with tolerance epsilon. Following
11561  /// this, we do up to max_adapt spatial adaptions (without
11562  /// re-examining the temporal error). If first==true, the initial conditions
11563  /// are re-assigned after the mesh adaptations.
11564  /// Shifting of time can be suppressed by overwriting the
11565  /// default value of shift (true). [Shifting must be done
11566  /// if first_timestep==true because we're constantly re-assigning
11567  /// the initial conditions; if first_timestep==true and shift==false
11568  /// shifting is performed anyway and a warning is issued.
11569  /// Pseudo-Boolean flag suppress_resolve_after_spatial_adapt [0: false;
11570  /// 1: true] does what it says.]
11571  //========================================================================
11573  const double& dt_desired,
11574  const double& epsilon,
11575  const unsigned& max_adapt,
11576  const unsigned& suppress_resolve_after_spatial_adapt_flag,
11577  const bool& first,
11578  const bool& shift_values)
11579  {
11580  // Store the initial time
11581  double initial_time = time_pt()->time();
11582 
11583  // Take adaptive timestep, adjusting dt until tolerance is satisfied
11584  double new_dt =
11585  adaptive_unsteady_newton_solve(dt_desired, epsilon, shift_values);
11586  double dt_taken = time_pt()->dt();
11587  oomph_info << "Accepted solution taken with timestep: " << dt_taken
11588  << std::endl;
11589 
11590 
11591  // Bail out straightaway if no spatial adaptation allowed
11592  if (max_adapt == 0)
11593  {
11594  oomph_info << "No spatial refinement allowed; max_adapt=0\n";
11595  return new_dt;
11596  }
11597 
11598  // Adapt problem/mesh
11599  unsigned n_refined = 0;
11600  unsigned n_unrefined = 0;
11601  adapt(n_refined, n_unrefined);
11602 
11603  // Check if mesh has been adapted on other processors
11604  Vector<int> total_ref_count(2);
11605  total_ref_count[0] = n_refined;
11606  total_ref_count[1] = n_unrefined;
11607 
11608 
11609 #ifdef OOMPH_HAS_MPI
11611  {
11612  // Sum n_refine across all processors
11613  Vector<int> ref_count(2);
11614  ref_count[0] = n_refined;
11615  ref_count[1] = n_unrefined;
11616  MPI_Allreduce(&ref_count[0],
11617  &total_ref_count[0],
11618  2,
11619  MPI_INT,
11620  MPI_SUM,
11621  communicator_pt()->mpi_comm());
11622  }
11623 #endif
11624 
11625 
11626  // Re-solve the problem if the adaptation has changed anything
11627  if ((total_ref_count[0] != 0) || (total_ref_count[1] != 0))
11628  {
11629  if (suppress_resolve_after_spatial_adapt_flag == 1)
11630  {
11631  oomph_info << "Mesh was adapted but re-solve has been suppressed."
11632  << std::endl;
11633  }
11634  else
11635  {
11636  oomph_info
11637  << "Mesh was adapted --> we'll re-solve for current timestep."
11638  << std::endl;
11639 
11640  // Reset time to what it was when we entered here
11641  // because it will be incremented again by dt_taken.
11642  time_pt()->time() = initial_time;
11643 
11644  // Shift the timesteps? No! They've been shifted already when we
11645  // called the solve with pure temporal adaptivity...
11646  bool shift = false;
11647 
11648  // Reset the inital condition on refined meshes
11649  if (first)
11650  {
11651  // Reset default set_initial_condition has been called flag to false
11653 
11654  // Reset the initial conditions
11655  oomph_info << "Re-assigning initial condition at time="
11656  << time_pt()->time() << std::endl;
11658 
11659  // This is the first timestep so shifting
11660  // has to be done following the assignment of initial conditions,
11661  // providing the default set_initial_condition function has not
11662  // been called.
11663  // In fact, unsteady_newton_solve(...) does that automatically.
11664  // We're changing the flag here to avoid warning messages.
11666  {
11667  shift = true;
11668  }
11669  }
11670 
11671  // Now take the step again on the refined mesh, using the same
11672  // timestep as used before.
11673  unsteady_newton_solve(dt_taken, max_adapt, first, shift);
11674  }
11675  }
11676  else
11677  {
11678  oomph_info << "Mesh wasn't adapted --> we'll accept spatial refinement."
11679  << std::endl;
11680  }
11681 
11682  return new_dt;
11683  }
11684 
11685 
11686  //========================================================================
11687  /// Initialise the previous values of the variables for time stepping
11688  /// corresponding to an impulsive start. Previous history for all data
11689  /// is generated by the appropriate timesteppers. Previous nodal
11690  /// positions are simply copied backwards.
11691  //========================================================================
11693  {
11694  // Assign the impulsive values in the "master" mesh
11696 
11697  // Loop over global data
11698  unsigned Nglobal = Global_data_pt.size();
11699  for (unsigned iglobal = 0; iglobal < Nglobal; iglobal++)
11700  {
11701  Global_data_pt[iglobal]
11702  ->time_stepper_pt()
11703  ->assign_initial_values_impulsive(Global_data_pt[iglobal]);
11704  }
11705  }
11706 
11707 
11708  //=======================================================================
11709  /// Assign the values for an impulsive start and also set the initial
11710  /// values of the previous dts to both be dt
11711  //======================================================================
11713  {
11714  // First initialise the dts and set the weights
11715  initialise_dt(dt);
11716  // Now call assign_initial_values_impulsive
11718  }
11719 
11720  //=======================================================================
11721  /// Return the current value of continuous time. If not Time object
11722  /// has been assigned, then throw an error
11723  //======================================================================
11724  double& Problem::time()
11725  {
11726  if (Time_pt == 0)
11727  {
11728  throw OomphLibError("Time object has not been set",
11729  OOMPH_CURRENT_FUNCTION,
11730  OOMPH_EXCEPTION_LOCATION);
11731  }
11732  else
11733  {
11734  return Time_pt->time();
11735  }
11736  }
11737 
11738  //=======================================================================
11739  /// Return the current value of continuous time. If not Time object
11740  /// has been assigned, then throw an error. Const version.
11741  //======================================================================
11742  double Problem::time() const
11743  {
11744  if (Time_pt == 0)
11745  {
11746  throw OomphLibError("Time object has not been set",
11747  OOMPH_CURRENT_FUNCTION,
11748  OOMPH_EXCEPTION_LOCATION);
11749  }
11750  else
11751  {
11752  return Time_pt->time();
11753  }
11754  }
11755 
11756 
11757  //=======================================================================
11758  /// Set all problem data to have the same timestepper (timestepper_pt).
11759  /// This is mainly used in continuation and bifurcation detection problems
11760  /// in which case the total number of unknowns may change and the changes
11761  /// to the underlying memory layout means that the Dof_pt must be
11762  /// reallocated. Thus, the function calls assign_eqn_numbers() and returns
11763  /// the number of new equation numbers.
11764  //=========================================================================
11766  TimeStepper* const& time_stepper_pt, const bool& preserve_existing_data)
11767  {
11768  // Set the timestepper for the master mesh's nodal and elemental data
11769  // to be the
11770  // continuation time stepper. This will wipe all storage other than
11771  // the 0th (present time) value at all the data objects
11773  preserve_existing_data);
11774 
11775  // Deal with the any additional mesh level timestepper data separately
11776  const unsigned n_sub_mesh = this->nsub_mesh();
11777  // If there is only one mesh
11778  if (n_sub_mesh == 0)
11779  {
11781  preserve_existing_data);
11782  }
11783  // Otherwise loop over the sub meshes
11784  else
11785  {
11786  // Assign global equation numbers first
11787  for (unsigned i = 0; i < n_sub_mesh; i++)
11788  {
11789  this->Sub_mesh_pt[i]->set_mesh_level_time_stepper(
11790  time_stepper_pt, preserve_existing_data);
11791  }
11792  }
11793 
11794  // Also set time stepper for global data
11795  const unsigned n_global = Global_data_pt.size();
11796  for (unsigned i = 0; i < n_global; ++i)
11797  {
11798  Global_data_pt[i]->set_time_stepper(time_stepper_pt,
11799  preserve_existing_data);
11800  }
11801 
11802  // We now need to reassign equations numbers because the Dof pointer
11803  // will be inappropriate because memory has been reallocated
11804 
11805 #ifdef OOMPH_HAS_MPI
11807  {
11808  std::ostringstream warning_stream;
11809  warning_stream << "This has not been comprehensively tested for "
11810  "distributed problems.\n"
11811  << "I'm sure that I need to worry about external halo and "
11812  "external elements."
11813  << std::endl;
11815  warning_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
11816  }
11817 
11818 #endif
11819 
11820  return (this->assign_eqn_numbers());
11821  }
11822 
11823 
11824  //========================================================================
11825  /// Shift all time-dependent data along for next timestep.
11826  //========================================================================
11828  {
11829  // Move the values of dt in the Time object
11830  Time_pt->shift_dt();
11831 
11832  // Only shift time values in the "master" mesh, otherwise things will
11833  // get shifted twice in complex problems
11835 
11836  // Shift global data with their own timesteppers
11837  unsigned Nglobal = Global_data_pt.size();
11838  for (unsigned iglobal = 0; iglobal < Nglobal; iglobal++)
11839  {
11840  Global_data_pt[iglobal]->time_stepper_pt()->shift_time_values(
11841  Global_data_pt[iglobal]);
11842  }
11843  }
11844 
11845 
11846  //========================================================================
11847  /// Calculate the predictions of all variables in problem
11848  //========================================================================
11850  {
11851 // Check that if we have multiple time steppers none of them want to
11852 // predict by calling an explicit timestepper (as opposed to doing
11853 // something like an explicit step by combining known history values, as
11854 // done in BDF).
11855 #ifdef PARANOID
11856  if (Time_stepper_pt.size() != 1)
11857  {
11858  for (unsigned j = 0; j < Time_stepper_pt.size(); j++)
11859  {
11860  if (time_stepper_pt()->predict_by_explicit_step())
11861  {
11862  std::string err = "Prediction by explicit step only works for "
11863  "problems with a simple time";
11864  err += "stepper. I think implementing anything more general will";
11865  err += "require a rewrite of explicit time steppers. - David";
11866  throw OomphLibError(
11867  err, OOMPH_EXCEPTION_LOCATION, OOMPH_CURRENT_FUNCTION);
11868  }
11869  }
11870  }
11871 #endif
11872 
11873 
11874  // Predict using an explicit timestepper (don't do it if adaptive = false
11875  // because pointers probably aren't set up).
11876  if (time_stepper_pt()->predict_by_explicit_step() &&
11877  time_stepper_pt()->adaptive_flag())
11878  {
11879  // Copy the time stepper's predictor pt into problem's explicit time
11880  // stepper pt (unless problem already has its own explicit time
11881  // stepper).
11883 #ifdef PARANOID
11884  if (ets_pt == 0)
11885  {
11886  std::string err = "Requested predictions by explicit step but explicit";
11887  err += " predictor pt is null.";
11888  throw OomphLibError(
11889  err, OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
11890  }
11891 
11892  if ((explicit_time_stepper_pt() != ets_pt) &&
11893  (explicit_time_stepper_pt() != 0))
11894  {
11895  throw OomphLibError("Problem has explicit time stepper other than "
11896  "predictor, not sure how to handle this yet ??ds",
11897  OOMPH_EXCEPTION_LOCATION,
11898  OOMPH_CURRENT_FUNCTION);
11899  }
11900 #endif
11901  explicit_time_stepper_pt() = ets_pt;
11902 
11903  // Backup dofs and time
11905 
11906 #ifdef PARANOID
11907  double backup_time = time();
11908 #endif
11909 
11910  // Move time back so that we are at the start of the timestep (as
11911  // explicit_timestep functions expect). This is needed because the
11912  // predictor calculations are done after unsteady newton solve has
11913  // started, and it has already moved time forwards.
11914  double dt = time_pt()->dt();
11915  time() -= dt;
11916 
11917  // Explicit step
11918  this->explicit_timestep(dt, false);
11919 
11920  // Copy predicted dofs and time to their storage slots.
11921  set_dofs(time_stepper_pt()->predictor_storage_index(), Dof_pt);
11923 
11924  // Check we got the times right
11925 #ifdef PARANOID
11926  if (std::abs(time() - backup_time) > 1e-12)
11927  {
11928  using namespace StringConversion;
11929  std::string err = "Predictor landed at the wrong time!";
11930  err += " Expected time " + to_string(backup_time, 14) + " but got ";
11931  err += to_string(time(), 14);
11932  throw OomphLibError(
11933  err, OOMPH_EXCEPTION_LOCATION, OOMPH_CURRENT_FUNCTION);
11934  }
11935 #endif
11936 
11937  // Restore dofs and time
11939  }
11940 
11941  // Otherwise we can do predictions in a more object oriented way using
11942  // whatever timestepper the data provides (this is the normal case).
11943  else
11944  {
11945  // Calculate all predictions in the "master" mesh
11947 
11948  // Calculate predictions for global data with their own timesteppers
11949  unsigned Nglobal = Global_data_pt.size();
11950  for (unsigned iglobal = 0; iglobal < Nglobal; iglobal++)
11951  {
11952  Global_data_pt[iglobal]->time_stepper_pt()->calculate_predicted_values(
11953  Global_data_pt[iglobal]);
11954  }
11955  }
11956 
11957  // If requested then copy the predicted value into the current time data
11958  // slots, ready for the newton solver to use as an initial guess.
11960  {
11961  // Not sure I know enough about distributed problems to implement
11962  // this. Probably you just need to loop over ndof_local or something,
11963  // but I can't really test it...
11964 #ifdef OOMPH_HAS_MPI
11965  if (distributed())
11966  {
11967  throw OomphLibError("Not yet implemented for distributed problems",
11968  OOMPH_EXCEPTION_LOCATION,
11969  OOMPH_CURRENT_FUNCTION);
11970  }
11971 #endif
11972 
11973  // With multiple time steppers this is much more complex becuase you
11974  // need to check the time stepper for each data to get the
11975  // predictor_storage_index(). Do-able if you need it though.
11976  if (Time_stepper_pt.size() != 1)
11977  {
11978  std::string err = "Not implemented for multiple time steppers";
11979  throw OomphLibError(
11980  err, OOMPH_EXCEPTION_LOCATION, OOMPH_CURRENT_FUNCTION);
11981  }
11982 
11983  // Get predicted values
11984  DoubleVector predicted_dofs;
11985  get_dofs(time_stepper_pt()->predictor_storage_index(), predicted_dofs);
11986 
11987  // Update dofs at current step
11988  for (unsigned i = 0; i < ndof(); i++)
11989  {
11990  dof(i) = predicted_dofs[i];
11991  }
11992  }
11993  }
11994 
11995  //======================================================================
11996  /// Enable recycling of the mass matrix in explicit timestepping
11997  /// schemes. Useful for timestepping on fixed meshes when you want
11998  /// to avoid the linear solve phase.
11999  //=====================================================================
12001  {
12004 
12005  // If we have a discontinuous formulation set the elements to reuse
12006  // their own mass matrices
12008  {
12009  const unsigned n_element = Problem::mesh_pt()->nelement();
12010  // Loop over the other elements
12011  for (unsigned e = 0; e < n_element; e++)
12012  {
12013  // Cache the element
12014  DGElement* const elem_pt =
12015  dynamic_cast<DGElement*>(Problem::mesh_pt()->element_pt(e));
12016  elem_pt->enable_mass_matrix_reuse();
12017  }
12018  }
12019  }
12020 
12021  //======================================================================
12022  /// Turn off the recyling of the mass matrix in explicit
12023  /// time-stepping schemes
12024  //======================================================================
12026  {
12029 
12030  // If we have a discontinuous formulation set the element-level
12031  // function
12033  {
12034  const unsigned n_element = Problem::mesh_pt()->nelement();
12035  // Loop over the other elements
12036  for (unsigned e = 0; e < n_element; e++)
12037  {
12038  // Cache the element
12039  DGElement* const elem_pt =
12040  dynamic_cast<DGElement*>(Problem::mesh_pt()->element_pt(e));
12041  elem_pt->disable_mass_matrix_reuse();
12042  }
12043  }
12044  }
12045 
12046 
12047  //=========================================================================
12048  /// Copy Data values, nodal positions etc from specified problem.
12049  /// Note: This is not a copy constructor. We assume that the current
12050  /// and the "original" problem have both been created by calling
12051  /// the same problem constructor so that all Data objects,
12052  /// time steppers etc. in the two problems are completely independent.
12053  /// This function copies the nodal, internal and global values
12054  /// and the time parameters from the original problem into "this"
12055  /// one. This functionality is required, e.g. for
12056  /// multigrid computations.
12057  //=========================================================================
12058  void Problem::copy(Problem* orig_problem_pt)
12059  {
12060  // Copy time
12061  //----------
12062 
12063  // Flag to indicate that orig problem is unsteady problem
12064  bool unsteady_flag = (orig_problem_pt->time_pt() != 0);
12065 
12066  // Copy current time and previous time increments for proper unsteady run
12067  if (unsteady_flag)
12068  {
12069  oomph_info << "Copying an unsteady problem." << std::endl;
12070  // Current time
12071  this->time_pt()->time() = orig_problem_pt->time_pt()->time();
12072  // Timesteps
12073  unsigned n_dt = orig_problem_pt->time_pt()->ndt();
12074  time_pt()->resize(n_dt);
12075  for (unsigned i = 0; i < n_dt; i++)
12076  {
12077  time_pt()->dt(i) = orig_problem_pt->time_pt()->dt(i);
12078  }
12079 
12080  // Find out how many timesteppers there are
12081  unsigned n_time_steppers = ntime_stepper();
12082 
12083  // Loop over them all and set the weights
12084  for (unsigned i = 0; i < n_time_steppers; i++)
12085  {
12087  }
12088  }
12089 
12090  // Copy nodes
12091  //-----------
12092 
12093  // Loop over submeshes:
12094  unsigned nmesh = nsub_mesh();
12095  if (nmesh == 0) nmesh = 1;
12096  for (unsigned m = 0; m < nmesh; m++)
12097  {
12098  // Find number of nodes in present mesh
12099  unsigned long n_node = mesh_pt(m)->nnode();
12100 
12101  // Check # of nodes:
12102  unsigned long n_node_orig = orig_problem_pt->mesh_pt(m)->nnode();
12103  if (n_node != n_node_orig)
12104  {
12105  std::ostringstream error_message;
12106  error_message << "Number of nodes in copy " << n_node
12107  << " not equal to the number in the original "
12108  << n_node_orig << std::endl;
12109 
12110  throw OomphLibError(error_message.str(),
12111  OOMPH_CURRENT_FUNCTION,
12112  OOMPH_EXCEPTION_LOCATION);
12113  }
12114 
12115  // Loop over the nodes
12116  for (unsigned long i = 0; i < n_node; i++)
12117  {
12118  // Try to cast to elastic node
12119  SolidNode* el_node_pt =
12120  dynamic_cast<SolidNode*>(mesh_pt(m)->node_pt(i));
12121  if (el_node_pt != 0)
12122  {
12123  SolidNode* el_node_orig_pt =
12124  dynamic_cast<SolidNode*>(orig_problem_pt->mesh_pt(m)->node_pt(i));
12125  el_node_pt->copy(el_node_orig_pt);
12126  }
12127  else
12128  {
12129  mesh_pt(m)->node_pt(i)->copy(orig_problem_pt->mesh_pt(m)->node_pt(i));
12130  }
12131  }
12132  }
12133 
12134 
12135  // Copy global data:
12136  //------------------
12137 
12138  // Number of global data
12139  unsigned n_global = Global_data_pt.size();
12140 
12141  // Check # of nodes in orig problem
12142  unsigned long n_global_orig = orig_problem_pt->nglobal_data();
12143  if (n_global != n_global_orig)
12144  {
12145  std::ostringstream error_message;
12146  error_message << "Number of global data in copy " << n_global
12147  << " not equal to the number in the original "
12148  << n_global_orig << std::endl;
12149 
12150  throw OomphLibError(
12151  error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
12152  }
12153 
12154  for (unsigned iglobal = 0; iglobal < n_global; iglobal++)
12155  {
12156  Global_data_pt[iglobal]->copy(orig_problem_pt->global_data_pt(iglobal));
12157  }
12158 
12159 
12160  // Copy internal data of elements:
12161  //--------------------------------
12162 
12163  // Loop over submeshes:
12164  for (unsigned m = 0; m < nmesh; m++)
12165  {
12166  // Loop over elements and deal with internal data
12167  unsigned n_element = mesh_pt(m)->nelement();
12168  for (unsigned e = 0; e < n_element; e++)
12169  {
12170  GeneralisedElement* el_pt = mesh_pt(m)->element_pt(e);
12171  unsigned n_internal = el_pt->ninternal_data();
12172  if (n_internal > 0)
12173  {
12174  // Check # of internals :
12175  unsigned long n_internal_orig =
12176  orig_problem_pt->mesh_pt(m)->element_pt(e)->ninternal_data();
12177  if (n_internal != n_internal_orig)
12178  {
12179  std::ostringstream error_message;
12180  error_message << "Number of internal data in copy " << n_internal
12181  << " not equal to the number in the original "
12182  << n_internal_orig << std::endl;
12183 
12184  throw OomphLibError(error_message.str(),
12185  OOMPH_CURRENT_FUNCTION,
12186  OOMPH_EXCEPTION_LOCATION);
12187  }
12188  for (unsigned i = 0; i < n_internal; i++)
12189  {
12190  el_pt->internal_data_pt(i)->copy(
12191  orig_problem_pt->mesh_pt(m)->element_pt(e)->internal_data_pt(i));
12192  }
12193  }
12194  }
12195  }
12196  }
12197 
12198  //=========================================================================
12199  /// Make and return a pointer to the copy of the problem. A virtual
12200  /// function that must be filled in by the user is they wish to perform
12201  /// adaptive refinement in bifurcation tracking or in multigrid problems.
12202  /// ALH: WILL NOT BE NECESSARY IN BIFURCATION TRACKING IN LONG RUN...
12203  //=========================================================================
12205  {
12206  std::ostringstream error_stream;
12207  error_stream
12208  << "This function must be overloaded in your specific problem, and must\n"
12209  << "create an exact copy of your problem. Usually this will be achieved\n"
12210  << "by a call to the constructor with exactly the same arguments as "
12211  "used\n";
12212 
12213  throw OomphLibError(
12214  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
12215  }
12216 
12217 
12218  //=========================================================================
12219  /// Dump refinement pattern of all refineable meshes and all generic
12220  /// Problem data to file for restart.
12221  //=========================================================================
12222  void Problem::dump(std::ofstream& dump_file) const
12223  {
12224  // Number of submeshes?
12225  unsigned n_mesh = nsub_mesh();
12226 
12227  dump_file << std::max(unsigned(1), n_mesh) << " # number of (sub)meshes "
12228  << std::endl;
12229 
12230  // Single mesh:
12231  //------------
12232  if (n_mesh == 0)
12233  {
12234  // Dump level of refinement before pruning
12235  if (TreeBasedRefineableMeshBase* mmesh_pt =
12236  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(0)))
12237  {
12238  dump_file << mmesh_pt->uniform_refinement_level_when_pruned()
12239  << " # uniform refinement when pruned " << std::endl;
12240  }
12241  else
12242  {
12243  dump_file << 0 << " # (fake) uniform refinement when pruned "
12244  << std::endl;
12245  }
12246  dump_file << 9999 << " # test flag for end of sub-meshes " << std::endl;
12247  }
12248 
12249  // Multiple submeshes
12250  //------------------
12251  else
12252  {
12253  // Loop over submeshes to dump level of refinement before pruning
12254  for (unsigned imesh = 0; imesh < n_mesh; imesh++)
12255  {
12256  if (TreeBasedRefineableMeshBase* mmesh_pt =
12257  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(imesh)))
12258  {
12259  dump_file << mmesh_pt->uniform_refinement_level_when_pruned()
12260  << " # uniform refinement when pruned " << std::endl;
12261  }
12262  else
12263  {
12264  dump_file << 0 << " # (fake) uniform refinement when pruned "
12265  << std::endl;
12266  }
12267  }
12268  dump_file << 9999 << " # test flag for end of sub-meshes " << std::endl;
12269  }
12270 
12271 #ifdef OOMPH_HAS_MPI
12272 
12273  const int my_rank = this->communicator_pt()->my_rank();
12274 
12275  // Record destination of all base elements
12276  unsigned n = Base_mesh_element_pt.size();
12277  Vector<int> local_base_element_processor(n, -1);
12278  Vector<int> base_element_processor(n, -1);
12279  for (unsigned e = 0; e < n; e++)
12280  {
12282  if (el_pt != 0)
12283  {
12284  if (!el_pt->is_halo())
12285  {
12286  local_base_element_processor[e] = my_rank;
12287  }
12288  }
12289  }
12290 
12291 
12292  // Get target for all base elements by reduction
12294  {
12295  // Check that the base elements have been associated to a processor
12296  // (the Base_mesh_elemen_pt is only used for structured meshes,
12297  // therefore, if there are no ustructured meshes as part of the
12298  // problem this container will be empty)
12299  if (n > 0)
12300  {
12301  MPI_Allreduce(&local_base_element_processor[0],
12302  &base_element_processor[0],
12303  n,
12304  MPI_INT,
12305  MPI_MAX,
12306  this->communicator_pt()->mpi_comm());
12307  }
12308  }
12309  else
12310  {
12311  // All the same...
12312  base_element_processor = local_base_element_processor;
12313  }
12314 
12315 
12316  dump_file << n << " # Number of base elements; partitioning follows.\n";
12317  for (unsigned e = 0; e < n; e++)
12318  {
12319  dump_file << base_element_processor[e] << "\n";
12320  }
12321  dump_file << "8888 #test flag for end of base element distribution\n";
12322 
12323 #endif
12324 
12325  // Single mesh:
12326  //------------
12327  if (n_mesh == 0)
12328  {
12329  // Dump single mesh refinement pattern (if mesh is refineable)
12330  if (TreeBasedRefineableMeshBase* mmesh_pt =
12331  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(0)))
12332  {
12333  mmesh_pt->dump_refinement(dump_file);
12334  }
12335 #ifdef OOMPH_HAS_TRIANGLE_LIB
12336  // Dump triangle mesh TriangulateIO which represents mesh topology
12337  TriangleMeshBase* mmesh_pt = dynamic_cast<TriangleMeshBase*>(mesh_pt(0));
12338  if (mmesh_pt != 0 && mmesh_pt->use_triangulateio_restart())
12339  {
12340 #ifdef OOMPH_HAS_MPI
12341  // Check if the mesh is distributed, if that is the case then
12342  // additional info. needs to be saved
12343  if (mmesh_pt->is_mesh_distributed())
12344  {
12345  // Dump the info. related with the distribution of the mesh
12346  mmesh_pt->dump_distributed_info_for_restart(dump_file);
12347  }
12348 #endif
12349  mmesh_pt->dump_triangulateio(dump_file);
12350  }
12351 #endif
12352  }
12353 
12354  // Multiple submeshes
12355  //------------------
12356  else
12357  {
12358  // Loop over submeshes
12359  for (unsigned imesh = 0; imesh < n_mesh; imesh++)
12360  {
12361  // Dump single mesh refinement pattern (if mesh is refineable)
12362  if (TreeBasedRefineableMeshBase* mmesh_pt =
12363  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(imesh)))
12364  {
12365  mmesh_pt->dump_refinement(dump_file);
12366  }
12367 #ifdef OOMPH_HAS_TRIANGLE_LIB
12368  // Dump triangle mesh TriangulateIO which represents mesh topology
12369  TriangleMeshBase* mmesh_pt =
12370  dynamic_cast<TriangleMeshBase*>(mesh_pt(imesh));
12371  if (mmesh_pt != 0 && mmesh_pt->use_triangulateio_restart())
12372  {
12373 #ifdef OOMPH_HAS_MPI
12374  // Check if the mesh is distributed, if that is the case then
12375  // additional info. needs to be saved
12376  if (mmesh_pt->is_mesh_distributed())
12377  {
12378  // Dump the info. related with the distribution of the mesh
12379  mmesh_pt->dump_distributed_info_for_restart(dump_file);
12380  }
12381 #endif
12382  mmesh_pt->dump_triangulateio(dump_file);
12383  }
12384 #endif
12385  } // End of loop over submeshes
12386  }
12387 
12388 
12389  // Dump time
12390  // ---------
12391 
12392  // Flag to indicate unsteady run
12393  bool unsteady_flag = (time_pt() != 0);
12394  dump_file << unsteady_flag << " # bool flag for unsteady" << std::endl;
12395 
12396  // Current time and previous time increments for proper unsteady run
12397  if (unsteady_flag)
12398  {
12399  // Current time
12400  dump_file << time_pt()->time() << " # Time " << std::endl;
12401  // Timesteps
12402  unsigned n_dt = time_pt()->ndt();
12403  dump_file << n_dt << " # Number of timesteps " << std::endl;
12404  for (unsigned i = 0; i < n_dt; i++)
12405  {
12406  dump_file << time_pt()->dt(i) << " # dt " << std::endl;
12407  }
12408  }
12409  // Dummy time and previous time increments for steady run
12410  else
12411  {
12412  // Current time
12413  dump_file << "0.0 # Dummy time from steady run " << std::endl;
12414  // Timesteps
12415  dump_file << "0 # Dummy number of timesteps from steady run" << std::endl;
12416  }
12417 
12418  // Loop over submeshes and dump their data
12419  unsigned nmesh = nsub_mesh();
12420  if (nmesh == 0) nmesh = 1;
12421  for (unsigned m = 0; m < nmesh; m++)
12422  {
12423  mesh_pt(m)->dump(dump_file);
12424  }
12425 
12426  // Dump global data
12427 
12428  // Loop over global data
12429  unsigned Nglobal = Global_data_pt.size();
12430  dump_file << Nglobal << " # number of global Data items " << std::endl;
12431  for (unsigned iglobal = 0; iglobal < Nglobal; iglobal++)
12432  {
12433  Global_data_pt[iglobal]->dump(dump_file);
12434  dump_file << std::endl;
12435  }
12436  }
12437 
12438  //=========================================================================
12439  /// Read refinement pattern of all refineable meshes and refine them
12440  /// accordingly, then read all Data and nodal position info from
12441  /// file for restart. Return flag to indicate if the restart was from
12442  /// steady or unsteady solution.
12443  //=========================================================================
12444  void Problem::read(std::ifstream& restart_file, bool& unsteady_restart)
12445  {
12446  // Check if the file is actually open as it won't be if it doesn't
12447  // exist! In that case we're almost certainly restarting the run on
12448  // a larger number of processors than the restart data was produced.
12449  // Say so and return
12450  bool restart_file_is_open = true;
12451  if (!restart_file.is_open())
12452  {
12453  std::ostringstream warn_message;
12454  warn_message << "Restart file isn't open -- I'm assuming that this is\n";
12455  warn_message << "because we're restarting on a larger number of\n";
12456  warn_message << "processor than were in use when the restart data was \n";
12457  warn_message << "dumped.\n";
12459  warn_message.str(), "Problem::read()", OOMPH_EXCEPTION_LOCATION);
12460  restart_file_is_open = false;
12461  }
12462 
12463  // Number of (sub)meshes?
12464  unsigned n_mesh = std::max(unsigned(1), nsub_mesh());
12465 
12466  std::string input_string;
12467 
12468  // Read line up to termination sign
12469  getline(restart_file, input_string, '#');
12470 
12471  // Ignore rest of line
12472  restart_file.ignore(80, '\n');
12473 
12474  // Read in number of sub-meshes
12475  unsigned n_submesh_read;
12476  n_submesh_read = std::atoi(input_string.c_str());
12477 
12478 #ifdef PARANOID
12479  if (restart_file_is_open)
12480  {
12481  if (n_submesh_read != n_mesh)
12482  {
12483  std::ostringstream error_message;
12484  error_message
12485  << "Number of sub-meshes specified in restart file, "
12486  << n_submesh_read << " doesn't \n match the my number of sub-meshes,"
12487  << n_mesh << std::endl
12488  << "Make sure all sub-meshes have been added to the global mesh\n"
12489  << "when calling the Problem::dump() function.\n";
12490  throw OomphLibError(error_message.str(),
12491  OOMPH_CURRENT_FUNCTION,
12492  OOMPH_EXCEPTION_LOCATION);
12493  }
12494  }
12495 #else
12496  // Suppress comiler warnings about non-used variable
12497  n_submesh_read++;
12498  n_submesh_read--;
12499 #endif
12500 
12501 
12502  // Read levels of refinement before pruning
12503 #ifdef OOMPH_HAS_MPI
12504  bool refine_and_prune_required = false;
12505 #endif
12506  Vector<unsigned> nrefinement_for_mesh(n_mesh);
12507  for (unsigned i = 0; i < n_mesh; i++)
12508  {
12509  // Read line up to termination sign
12510  getline(restart_file, input_string, '#');
12511 
12512  // Ignore rest of line
12513  restart_file.ignore(80, '\n');
12514 
12515  // Convert
12516  nrefinement_for_mesh[i] = std::atoi(input_string.c_str());
12517 
12518  // Get pointer to sub-mesh in incarnation as tree-based refineable mesh
12519  TreeBasedRefineableMeshBase* ref_mesh_pt =
12520  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(i));
12521 
12522  // If it's not a tree-based refineable mesh, ignore the following
12523  if (ref_mesh_pt == 0)
12524  {
12525  if (nrefinement_for_mesh[i] != 0)
12526  {
12527  std::ostringstream error_stream;
12528  error_stream << "Nonzero uniform-refinement-when-pruned specified\n"
12529  << "even though mesh is not tree-based. Odd. May want\n"
12530  << "to check this carefully before disabling this \n"
12531  << "warning/error -- most likely if/when we start to\n"
12532  << "prune unstructured meshes [though I can't see why\n"
12533  << "we would want to do this, given that they are \n"
12534  << "currently totally re-generated...]\n";
12535  throw OomphLibError(error_stream.str(),
12536  OOMPH_CURRENT_FUNCTION,
12537  OOMPH_EXCEPTION_LOCATION);
12538  }
12539  }
12540  else
12541  {
12542  // Get min and max refinement level
12543  unsigned local_min_ref = 0;
12544  unsigned local_max_ref = 0;
12545  ref_mesh_pt->get_refinement_levels(local_min_ref, local_max_ref);
12546 
12547  // Overall min refinement level over all meshes
12548  unsigned min_ref = local_min_ref;
12549 
12550 #ifdef OOMPH_HAS_MPI
12552  {
12553  // Reconcile between processors: If (e.g. following
12554  // distribution/pruning) the mesh has no elements on this
12555  // processor) then ignore its contribution to the poll of
12556  // max/min refinement levels
12557  int int_local_min_ref = local_min_ref;
12558  if (ref_mesh_pt->nelement() == 0)
12559  {
12560  int_local_min_ref = INT_MAX;
12561  }
12562  int int_min_ref = 0;
12563  MPI_Allreduce(&int_local_min_ref,
12564  &int_min_ref,
12565  1,
12566  MPI_INT,
12567  MPI_MIN,
12568  Communicator_pt->mpi_comm());
12569 
12570  // Overall min refinement level over all meshes
12571  min_ref = unsigned(int_min_ref);
12572  }
12573 #endif
12574 
12575  // Need to refine less
12576  if (nrefinement_for_mesh[i] >= min_ref)
12577  {
12578  nrefinement_for_mesh[i] -= min_ref;
12579  }
12580  }
12581 
12582 #ifdef OOMPH_HAS_MPI
12583  if (nrefinement_for_mesh[i] > 0)
12584  {
12585  refine_and_prune_required = true;
12586  }
12587 #endif
12588  }
12589 
12590 
12591  // Reconcile overall need to refine and prune (even empty
12592  // processors have to participate in some communication!)
12593 #ifdef OOMPH_HAS_MPI
12595  {
12596  unsigned local_req_flag = 0;
12597  unsigned req_flag = 0;
12598  if (refine_and_prune_required)
12599  {
12600  local_req_flag = 1;
12601  }
12602  MPI_Allreduce(&local_req_flag,
12603  &req_flag,
12604  1,
12605  MPI_UNSIGNED,
12606  MPI_MAX,
12607  Communicator_pt->mpi_comm());
12608  refine_and_prune_required = false;
12609  if (req_flag == 1)
12610  {
12611  refine_and_prune_required = true;
12612  }
12613 
12614  // If refine and prune is required make number of uniform
12615  // refinements for each mesh consistent otherwise code
12616  // hangs on "empty" processors for which no restart file exists
12617  if (refine_and_prune_required)
12618  {
12619  // This is what we have locally
12620  Vector<unsigned> local_nrefinement_for_mesh(nrefinement_for_mesh);
12621  // Synchronise over all processors with max operation
12622  MPI_Allreduce(&local_nrefinement_for_mesh[0],
12623  &nrefinement_for_mesh[0],
12624  n_mesh,
12625  MPI_UNSIGNED,
12626  MPI_MAX,
12627  Communicator_pt->mpi_comm());
12628 
12629 #ifdef PARANOID
12630  // Check it: Reconciliation should only be required for
12631  // for processors on which no restart file was opened and
12632  // for which the meshes are therefore empty
12633  bool fail = false;
12634  std::ostringstream error_message;
12635  error_message << "Number of uniform refinements was not consistent \n"
12636  << "for following meshes during restart on processor \n"
12637  << "on which restart file could be opened:\n";
12638  for (unsigned i = 0; i < n_mesh; i++)
12639  {
12640  if ((local_nrefinement_for_mesh[i] != nrefinement_for_mesh[i]) &&
12641  restart_file_is_open)
12642  {
12643  fail = true;
12644  error_message << "Sub-mesh: " << i << "; local nrefinement: "
12645  << local_nrefinement_for_mesh[i] << " "
12646  << "; global/synced nrefinement: "
12647  << nrefinement_for_mesh[i] << "\n";
12648  }
12649  }
12650  if (fail)
12651  {
12653  error_message.str(), "Problem::read()", OOMPH_EXCEPTION_LOCATION);
12654  }
12655 #endif
12656  }
12657  }
12658 #endif
12659 
12660  // Read line up to termination sign
12661  getline(restart_file, input_string, '#');
12662 
12663  // Ignore rest of line
12664  restart_file.ignore(80, '\n');
12665 
12666  // Check flag that indicates that we've read the final data
12667  unsigned tmp;
12668  tmp = std::atoi(input_string.c_str());
12669 
12670 #ifdef PARANOID
12671  if (restart_file_is_open)
12672  {
12673  if (tmp != 9999)
12674  {
12675  std::ostringstream error_message;
12676  error_message
12677  << "Error in reading restart data: Uniform refinement when pruned \n"
12678  << "flags should be followed by 9999.\n";
12679  throw OomphLibError(error_message.str(),
12680  OOMPH_CURRENT_FUNCTION,
12681  OOMPH_EXCEPTION_LOCATION);
12682  }
12683  }
12684 
12685 #else
12686  // Suppress comiler warnings about non-used variable
12687  tmp++;
12688  tmp--;
12689 #endif
12690 
12691 
12692 #ifdef OOMPH_HAS_MPI
12693 
12694  // Refine and prune if required
12695  if (refine_and_prune_required)
12696  {
12697  refine_uniformly(nrefinement_for_mesh);
12699  }
12700 
12701  // target_domain_for_local_non_halo_element[e] contains the number
12702  // of the domain [0,1,...,nproc-1] to which non-halo element e on THE
12703  // CURRENT PROCESSOR ONLY has been assigned. The order of the non-halo
12704  // elements is the same as in the Problem's mesh, with the halo
12705  // elements being skipped.
12706  Vector<unsigned> target_domain_for_local_non_halo_element;
12707 
12708  // If a restart file has been generated using code compiled without MPI
12709  // then it will not have any of the base element data.
12710  // If we try to read in that file with code that has been compied using
12711  // MPI, even if running only one processor, then it will fail here.
12712  // The ideal fix is to edit the restart file so that it contains the two
12713  // lines
12714  //
12715  // 0 # Number of base elements; partitioning follows.
12716  // 8888 # Test flag for end of base element distribution
12717  //
12718  // after the end of the sub-meshes, but before the number of elements
12719  // However, we can determine that this is the problem if n_base = 0,
12720  // so there is a little bit of logic below to catch this case
12721 
12722  // Store current location in the file (before we are about to read
12723  // in either the base mesh or number of elements of the first mesh)
12724  std::streampos position_before_base_element = restart_file.tellg();
12725  // Boolean flag used to set whether to read in base element info
12726  bool read_in_base_element_info = true;
12727 
12728  // Read line up to termination sign
12729  getline(restart_file, input_string, '#');
12730 
12731  // Ignore rest of line
12732  restart_file.ignore(80, '\n');
12733 
12734  // Get number of base elements as recorded
12735  unsigned n_base_element_read_in = atoi(input_string.c_str());
12736  unsigned nbase = Base_mesh_element_pt.size();
12737  if (restart_file_is_open)
12738  {
12739  if (n_base_element_read_in != nbase)
12740  {
12741  // If we have zero base elements the problem could be that the
12742  // restart file was generated without MPI. Issue a warning
12743  // and continue anyway
12744  if (nbase == 0)
12745  {
12746  std::ostringstream warn_message;
12747  warn_message
12748  << "The number of base elements in the mesh is 0,\n"
12749  << " but the restart file indicates that there are "
12750  << n_base_element_read_in << ".\n"
12751  << "This could be because the restart file was \n"
12752  << "generated by using code without MPI.\n"
12753  << "\n"
12754  << "The best fix is to include two additional lines\n"
12755  << "in the restart file: \n\n"
12756  << "0 # Number of base elements; partitioning follows.\n"
12757  << "8888 # Test flag for end of base element distribution\n"
12758  << "\n"
12759  << "These lines go after the flag 9999 that indicates\n"
12760  << "the end of the submesh information.\n"
12761  << "\n"
12762  << "The file will now continue to be read assuming that\n"
12763  << "the base element information is not present.\n"
12764  << "If you get strange results then please look carefully\n"
12765  << "at the restart file. The safest thing to do is to \n"
12766  << "ensure that the restart file was generated by code\n"
12767  << "compiled and run with the same parallel options.\n";
12768  OomphLibWarning(warn_message.str(),
12769  OOMPH_CURRENT_FUNCTION,
12770  OOMPH_EXCEPTION_LOCATION);
12771  // Set the skip flag to true
12772  // and rewind the file pointer
12773  read_in_base_element_info = false;
12774  restart_file.seekg(position_before_base_element);
12775  }
12776  // Otherwise throw a hard error
12777  else
12778  {
12779  std::ostringstream error_message;
12780  error_message << "About to read " << n_base_element_read_in
12781  << " base elements \n"
12782  << "though we only have " << nbase
12783  << " base elements in mesh.\n";
12784  throw OomphLibError(error_message.str(),
12785  OOMPH_CURRENT_FUNCTION,
12786  OOMPH_EXCEPTION_LOCATION);
12787  }
12788  }
12789  }
12790 
12791  // Read in the remaning base element information, if necessary
12792  if (read_in_base_element_info == true)
12793  {
12794  // Read in target_domain_for_base_element[e] for all base elements
12795  Vector<unsigned> target_domain_for_base_element(nbase);
12796  for (unsigned e = 0; e < nbase; e++)
12797  {
12798  // Read line
12799  getline(restart_file, input_string);
12800 
12801  // Get target domain
12802  target_domain_for_base_element[e] = atoi(input_string.c_str());
12803  }
12804 
12805  // Read line up to termination sign
12806  getline(restart_file, input_string, '#');
12807 
12808  // Ignore rest of line
12809  restart_file.ignore(80, '\n');
12810 
12811  // Check flag that indicates that we've read the final data
12812  tmp = std::atoi(input_string.c_str());
12813 
12814 
12815 #ifdef PARANOID
12816  if (restart_file_is_open)
12817  {
12818  if (tmp != 8888)
12819  {
12820  std::ostringstream error_message;
12821  error_message
12822  << "Error in reading restart data: Target proc for base elements \n"
12823  << "should be followed by 8888.\n";
12824  throw OomphLibError(error_message.str(),
12825  OOMPH_CURRENT_FUNCTION,
12826  OOMPH_EXCEPTION_LOCATION);
12827  }
12828  }
12829 #endif
12830 
12831  // Loop over all elements (incl. any FaceElements) and assign
12832  // target domain for all local non-halo elements and check if
12833  // load balancing is required -- no need to do this if problem is
12834  // not distributed.
12835  unsigned load_balance_required_flag = 0;
12837  {
12838  // Working with TreeBasedRefineableMeshBase mesh
12839  unsigned local_load_balance_required_flag = 0;
12840  if (dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(0)))
12841  {
12842  const int my_rank = this->communicator_pt()->my_rank();
12843  unsigned nel = mesh_pt()->nelement();
12844  for (unsigned e = 0; e < nel; e++)
12845  {
12846  GeneralisedElement* el_pt = mesh_pt()->element_pt(e);
12847  if (!el_pt->is_halo())
12848  {
12849  // Get element number (plus one) in base element enumeration
12850  unsigned el_number_in_base_mesh_plus_one =
12852 
12853  // If it's zero then we haven't found it, it may be a FaceElement
12854  // (in which case we move it to the same processor as its bulk
12855  // element
12856  if (el_number_in_base_mesh_plus_one == 0)
12857  {
12858  FaceElement* face_el_pt = dynamic_cast<FaceElement*>(el_pt);
12859  if (face_el_pt != 0)
12860  {
12861  // Get corresponding bulk element
12862  FiniteElement* bulk_el_pt = face_el_pt->bulk_element_pt();
12863 
12864  // Use its element number (plus one) in base element
12865  // enumeration
12866  el_number_in_base_mesh_plus_one =
12868 
12869  // If this is zero too we have a problem
12870  if (el_number_in_base_mesh_plus_one == 0)
12871  {
12872  throw OomphLibError(
12873  "el_number_in_base_mesh_plus_one=0 for bulk",
12874  "Problem::read()",
12875  OOMPH_EXCEPTION_LOCATION);
12876  }
12877  }
12878  }
12879 
12880  // If we've made it here then we're not dealing with a
12881  // FaceElement but with an element that doesn't exist locally
12882  // --> WTF?
12883  if (el_number_in_base_mesh_plus_one == 0)
12884  {
12885  throw OomphLibError("el_number_in_base_mesh_plus_one=0",
12886  OOMPH_CURRENT_FUNCTION,
12887  OOMPH_EXCEPTION_LOCATION);
12888  }
12889 
12890  // Assign target domain for next local non-halo element in
12891  // the order in which it's encountered in the global mesh
12892  target_domain_for_local_non_halo_element.push_back(
12893  target_domain_for_base_element[el_number_in_base_mesh_plus_one -
12894  1]);
12895 
12896  // Do elements on this processor to be moved elsewhere?
12897  if (int(target_domain_for_base_element
12898  [el_number_in_base_mesh_plus_one - 1]) != my_rank)
12899  {
12900  local_load_balance_required_flag = 1;
12901  }
12902  }
12903  }
12904 
12905  } // if (working with TreeBasedRefineableMeshBase mesh)
12906 
12907  // Get overall need to load balance by max
12908  MPI_Allreduce(&local_load_balance_required_flag,
12909  &load_balance_required_flag,
12910  1,
12911  MPI_UNSIGNED,
12912  MPI_MAX,
12913  this->communicator_pt()->mpi_comm());
12914  }
12915 
12916  // Do we need to load balance?
12917  if (load_balance_required_flag == 1)
12918  {
12919  oomph_info << "Doing load balancing after pruning\n";
12920  DocInfo doc_info;
12921  doc_info.disable_doc();
12922  bool report_stats = false;
12923  load_balance(
12924  doc_info, report_stats, target_domain_for_local_non_halo_element);
12925  oomph_info << "Done load balancing after pruning\n";
12926  }
12927  else
12928  {
12929  oomph_info << "No need for load balancing after pruning\n";
12930  }
12931  } // End of read in base element information
12932 #endif
12933 
12934 
12935  // Boolean to record if any unstructured bulk meshes have
12936  // been read in (and therefore completely re-generated, with new
12937  // elements and nodes) from disk
12938  bool have_read_unstructured_mesh = false;
12939 
12940  // Call the actions before adaptation
12942 
12943  // If there are unstructured meshes in the problem we need
12944  // to strip out any face elements that are attached to them
12945  // because restart of unstructured meshes re-creates their elements
12946  // and nodes from scratch, leading to dangling pointers from the
12947  // face elements to the old elements and nodes. This function is
12948  // virtual and (practically) empty in the Problem base class
12949  // but toggles a flag to indicate that it has been called. We can then
12950  // issue a warning below, prompting the user to consider overloading it
12951  // if the problem is found to contain unstructured bulk meshes.
12952  // Warning can be ignored if the bulk mesh is not associated with any
12953  // face elements.
12956 
12957  // Update number of submeshes
12958  n_mesh = nsub_mesh();
12959 
12960  // Single mesh:
12961  //------------
12962  if (n_mesh == 0)
12963  {
12964  // Refine single mesh (if it's refineable)
12965  if (TreeBasedRefineableMeshBase* mmesh_pt =
12966  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(0)))
12967  {
12968  // When we get in here the problem has been constructed
12969  // by the constructor and the mesh is its original unrefined
12970  // form.
12971  // RefineableMeshBase::refine(...) reads the refinement pattern from the
12972  // specified file and performs refinements until the mesh has
12973  // reached the same level of refinement as the mesh that existed
12974  // when the problem was dumped to disk.
12975  mmesh_pt->refine(restart_file);
12976  }
12977 #ifdef OOMPH_HAS_TRIANGLE_LIB
12978  // Regenerate mesh from triangulate IO if it's a triangular mesh
12979  TriangleMeshBase* mmesh_pt = dynamic_cast<TriangleMeshBase*>(mesh_pt(0));
12980  if (mmesh_pt != 0 && mmesh_pt->use_triangulateio_restart())
12981  {
12982 #ifdef OOMPH_HAS_MPI
12983  // Check if the mesh is distributed, if that is the case then
12984  // additional info. needs to be read
12985  if (mmesh_pt->is_mesh_distributed())
12986  {
12987  // Dump the info. related with the distribution of the mesh
12988  mmesh_pt->read_distributed_info_for_restart(restart_file);
12989  }
12990 #endif
12991  // The function reads the TriangulateIO data structure from the dump
12992  // file and then completely regenerates the mesh using the
12993  // data structure
12994  mmesh_pt->remesh_from_triangulateio(restart_file);
12995  have_read_unstructured_mesh = true;
12996 #ifdef OOMPH_HAS_MPI
12997  // Check if the mesh is distributed, if that is the case then we
12998  // need to re-establish the halo/haloed scheme (similar as in the
12999  // RefineableTriangleMesh::adapt() method)
13000  if (mmesh_pt->is_mesh_distributed())
13001  {
13003  this->communicator_pt(), restart_file);
13004  }
13005 #endif
13006  // Still left to update the polylines representation, that is performed
13007  // later since the nodes positions may still change when reading info.
13008  // for the mesh, see below
13009  }
13010 #endif
13011  }
13012 
13013  // Multiple submeshes
13014  //------------------
13015  else
13016  {
13017  // Loop over submeshes
13018  for (unsigned imesh = 0; imesh < n_mesh; imesh++)
13019  {
13020  // Refine single mesh (if its refineable)
13021  if (TreeBasedRefineableMeshBase* mmesh_pt =
13022  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(imesh)))
13023  {
13024  // When we get in here the problem has been constructed
13025  // by the constructor and the mesh is its original unrefined
13026  // form.
13027  // RefineableMeshBase::refine(...) reads the refinement pattern from
13028  // the specified file and performs refinements until the mesh has
13029  // reached the same level of refinement as the mesh that existed
13030  // when the problem was dumped to disk.
13031  mmesh_pt->refine(restart_file);
13032  }
13033 #ifdef OOMPH_HAS_TRIANGLE_LIB
13034  // Regenerate mesh from triangulate IO if it's a triangular mesh
13035  TriangleMeshBase* mmesh_pt =
13036  dynamic_cast<TriangleMeshBase*>(mesh_pt(imesh));
13037  if (mmesh_pt != 0 && mmesh_pt->use_triangulateio_restart())
13038  {
13039 #ifdef OOMPH_HAS_MPI
13040  // Check if the mesh is distributed, if that is the case then
13041  // additional info. needs to be read
13042  if (mmesh_pt->is_mesh_distributed())
13043  {
13044  // Dump the info. related with the distribution of the mesh
13045  mmesh_pt->read_distributed_info_for_restart(restart_file);
13046  }
13047 #endif
13048  // The function reads the TriangulateIO data structure from the dump
13049  // file and then completely regenerates the mesh using the
13050  // data structure
13051  mmesh_pt->remesh_from_triangulateio(restart_file);
13052  have_read_unstructured_mesh = true;
13053 
13054 #ifdef OOMPH_HAS_MPI
13055  // Check if the mesh is distributed, if that is the case then we
13056  // need to re-establish the halo/haloed scheme (similar as in the
13057  // RefineableTriangleMesh::adapt() method)
13058  if (mmesh_pt->is_mesh_distributed())
13059  {
13061  this->communicator_pt(), restart_file);
13062  }
13063 #endif
13064  // Still left to update the polylines representation, that is
13065  // performed later since the nodes positions may still change when
13066  // reading info. for the mesh, see below
13067  }
13068 #endif
13069  } // End of loop over submeshes
13070 
13071 
13072  // Rebuild the global mesh
13074  }
13075 
13076  // Any actions after adapt
13078 
13079  // Re-attach face elements (or whatever else needs to be done
13080  // following the total re-generation of the unstructured meshes
13083 
13084 
13085  // Issue warning:
13087  {
13088  if (have_read_unstructured_mesh)
13089  {
13092  {
13093  std::ostringstream warn_message;
13094  warn_message
13095  << "I've just read in some unstructured meshes and have, in\n"
13096  << "the process, totally re-generated their nodes and elements.\n"
13097  << "This may create dangling pointers that still point to the\n"
13098  << "old nodes and elements, e.g. because FaceElements were\n"
13099  << "attached to these meshes or pointers to nodes and elements\n"
13100  << "were stored somewhere. FaceElements should therefore be\n"
13101  << "removed before reading in these meshes, using an overloaded\n"
13102  << "version of the function\n\n"
13103  << " Problem::actions_before_read_unstructured_meshes()\n\n"
13104  << "and then re-attached using an overloaded version of\n\n"
13105  << " Problem::actions_after_read_unstructured_meshes().\n\n"
13106  << "The required content of these functions is likely to be "
13107  "similar\n"
13108  << "to the Problem::actions_before_adapt() and \n"
13109  << "Problem::actions_after_adapt() that would be required in\n"
13110  << "a spatially adaptive computation. If these functions already\n"
13111  << "exist and perform the required actions, the \n"
13112  << "actions_before/after_read_unstructured_meshes() functions\n"
13113  << "can remain empty because the former are called automatically.\n"
13114  << "In this case, this warning my be suppressed by setting the\n"
13115  << "public boolean\n\n"
13116  << " "
13117  "Problem::Suppress_warning_about_actions_before_read_"
13118  "unstructured_meshes\n\n"
13119  << "to true." << std::endl;
13120  OomphLibWarning(warn_message.str(),
13121  OOMPH_CURRENT_FUNCTION,
13122  OOMPH_EXCEPTION_LOCATION);
13123  }
13124  }
13125  }
13126 
13127  // Setup equation numbering scheme
13128  oomph_info << "\nNumber of equations in Problem::read(): "
13129  << assign_eqn_numbers() << std::endl
13130  << std::endl;
13131  // Read time info
13132  //---------------
13133  unsigned local_unsteady_restart_flag = 0;
13134  double local_time = -DBL_MAX;
13135  unsigned local_n_dt = 0;
13136 #ifdef OOMPH_HAS_MPI
13137  unsigned local_sync_needed_flag = 0;
13138 #endif
13139  Vector<double> local_dt;
13140 
13141  if (restart_file.is_open())
13142  {
13143  oomph_info << "Restart file exists" << std::endl;
13144 #ifdef OOMPH_HAS_MPI
13145  local_sync_needed_flag = 0;
13146 #endif
13147  // Read line up to termination sign
13148  getline(restart_file, input_string, '#');
13149 
13150  // Ignore rest of line
13151  restart_file.ignore(80, '\n');
13152 
13153  // Is the restart data from an unsteady run?
13154  local_unsteady_restart_flag = atoi(input_string.c_str());
13155 
13156  // Read line up to termination sign
13157  getline(restart_file, input_string, '#');
13158 
13159  // Ignore rest of line
13160  restart_file.ignore(80, '\n');
13161 
13162  // Read in initial time and set
13163  local_time = atof(input_string.c_str());
13164 
13165  // Read line up to termination sign
13166  getline(restart_file, input_string, '#');
13167 
13168  // Ignore rest of line
13169  restart_file.ignore(80, '\n');
13170 
13171  // Read & set number of timesteps
13172  local_n_dt = atoi(input_string.c_str());
13173  local_dt.resize(local_n_dt);
13174 
13175  // Read in timesteps:
13176  for (unsigned i = 0; i < local_n_dt; i++)
13177  {
13178  // Read line up to termination sign
13179  getline(restart_file, input_string, '#');
13180 
13181  // Ignore rest of line
13182  restart_file.ignore(80, '\n');
13183 
13184  // Read in initial time and set
13185  double prev_dt = atof(input_string.c_str());
13186  local_dt[i] = prev_dt;
13187  }
13188  }
13189  else
13190  {
13191  oomph_info << "Restart file does not exist" << std::endl;
13192 #ifdef OOMPH_HAS_MPI
13193  local_sync_needed_flag = 1;
13194 #endif
13195  }
13196 
13197 
13198  // No prepare global values, possibly via sync
13199  Vector<double> dt;
13200 
13201  // Do we need to sync?
13202  unsigned sync_needed_flag = 0;
13203 
13204 #ifdef OOMPH_HAS_MPI
13206  {
13207  // Get need to sync by max
13208  MPI_Allreduce(&local_sync_needed_flag,
13209  &sync_needed_flag,
13210  1,
13211  MPI_UNSIGNED,
13212  MPI_MAX,
13213  this->communicator_pt()->mpi_comm());
13214  }
13215 #endif
13216 
13217  // Synchronise
13218  if (sync_needed_flag == 1)
13219  {
13220 #ifdef OOMPH_HAS_MPI
13221 
13222 
13223 #ifdef PARANOID
13225  {
13226  std::ostringstream error_message;
13227  error_message << "Synchronisation of temporal restart data \n"
13228  << "required even though Problem hasn't been distributed "
13229  "-- very odd!\n";
13230  throw OomphLibError(error_message.str(),
13231  OOMPH_CURRENT_FUNCTION,
13232  OOMPH_EXCEPTION_LOCATION);
13233  }
13234 #endif
13235 
13236  // Get unsteady restart flag by max-based reduction
13237  unsigned unsteady_restart_flag = 0;
13238  MPI_Allreduce(&local_unsteady_restart_flag,
13239  &unsteady_restart_flag,
13240  1,
13241  MPI_UNSIGNED,
13242  MPI_MAX,
13243  this->communicator_pt()->mpi_comm());
13244 
13245  // So, is it an unsteady restart?
13246  unsteady_restart = false;
13247  if (unsteady_restart_flag == 1)
13248  {
13249  unsteady_restart = true;
13250 
13251  // Get time by max
13252  double time = -DBL_MAX;
13253  MPI_Allreduce(&local_time,
13254  &time,
13255  1,
13256  MPI_DOUBLE,
13257  MPI_MAX,
13258  this->communicator_pt()->mpi_comm());
13259  time_pt()->time() = time;
13260 
13261  // Get number of timesteps by max-based reduction
13262  unsigned n_dt = 0;
13263  MPI_Allreduce(&local_n_dt,
13264  &n_dt,
13265  1,
13266  MPI_UNSIGNED,
13267  MPI_MAX,
13268  this->communicator_pt()->mpi_comm());
13269 
13270  // Resize whatever needs resizing
13271  time_pt()->resize(n_dt);
13272  dt.resize(n_dt);
13273  if (local_dt.size() == 0)
13274  {
13275  local_dt.resize(n_dt, -DBL_MAX);
13276  }
13277 
13278  // Get timesteps increments by max-based reduction
13279  MPI_Allreduce(&local_dt[0],
13280  &dt[0],
13281  n_dt,
13282  MPI_DOUBLE,
13283  MPI_MAX,
13284  this->communicator_pt()->mpi_comm());
13285  }
13286 
13287 #else
13288 
13289  std::ostringstream error_message;
13290  error_message
13291  << "Synchronisation of temporal restart data \n"
13292  << "required even though we don't have mpi support -- very odd!\n";
13293  throw OomphLibError(
13294  error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
13295 
13296 #endif
13297  }
13298  // No sync needed -- just copy across
13299  else
13300  {
13301  unsteady_restart = false;
13302  if (local_unsteady_restart_flag == 1)
13303  {
13304  unsteady_restart = true;
13305  time_pt()->time() = local_time;
13306  time_pt()->resize(local_n_dt);
13307  dt.resize(local_n_dt);
13308  for (unsigned i = 0; i < local_n_dt; i++)
13309  {
13310  dt[i] = local_dt[i];
13311  }
13312  }
13313  }
13314 
13315  // Initialise timestep -- also sets the weights for all timesteppers
13316  // in the problem.
13317  if (unsteady_restart) initialise_dt(dt);
13318 
13319  // Loop over submeshes:
13320  unsigned nmesh = nsub_mesh();
13321  if (nmesh == 0) nmesh = 1;
13322  for (unsigned m = 0; m < nmesh; m++)
13323  {
13324  // //---------------------------------------------------------
13325  // // Keep this commented out code around to debug restarts
13326  // //---------------------------------------------------------
13327  // std::ofstream some_file;
13328  // char filename[100];
13329  // sprintf(filename,"read_mesh%i_on_proc%i.dat",m,
13330  // this->communicator_pt()->my_rank());
13331  // some_file.open(filename);
13332  // mesh_pt(m)->output(some_file);
13333  // some_file.close();
13334 
13335  // sprintf(filename,"read_mesh%i_with_haloes_on_proc%i.dat",m,
13336  // this->communicator_pt()->my_rank());
13337  // mesh_pt(m)->enable_output_of_halo_elements();
13338  // some_file.open(filename);
13339  // mesh_pt(m)->output(some_file);
13340  // mesh_pt(m)->disable_output_of_halo_elements();
13341  // some_file.close();
13342  // oomph_info << "Doced mesh " << m << " before reading\n";
13343 
13344  // sprintf(filename,"read_nodes_mesh%i_on_proc%i.dat",m,
13345  // this->communicator_pt()->my_rank());
13346  // some_file.open(filename);
13347  // unsigned nnod=mesh_pt(m)->nnode();
13348  // for (unsigned j=0;j<nnod;j++)
13349  // {
13350  // Node* nod_pt=mesh_pt(m)->node_pt(j);
13351  // unsigned n=nod_pt->ndim();
13352  // for (unsigned i=0;i<n;i++)
13353  // {
13354  // some_file << nod_pt->x(i) << " ";
13355  // }
13356  // some_file << nod_pt->is_halo() << " "
13357  // << nod_pt->nvalue() << " "
13358  // << nod_pt->hang_code() << "\n";
13359  // }
13360  // some_file.close();
13361  // oomph_info << "Doced mesh " << m << " before reading\n";
13362  // //---------------------------------------------------------
13363  // // End keep this commented out code around to debug restarts
13364  // //---------------------------------------------------------
13365 
13366  mesh_pt(m)->read(restart_file);
13367 
13368 #ifdef OOMPH_HAS_TRIANGLE_LIB
13369  // Here update the polyline representation if working with
13370  // triangle base meshes
13371  if (TriangleMeshBase* mmesh_pt =
13372  dynamic_cast<TriangleMeshBase*>(mesh_pt(m)))
13373  {
13374  // In charge of updating the polylines representation to the
13375  // current refinement/unrefinement level after restart, it
13376  // also update the shared boundaries in case of working with a
13377  // distributed mesh
13378  mmesh_pt->update_polyline_representation_from_restart();
13379  }
13380 #endif // #ifdef OOMPH_HAS_TRIANGLE_LIB
13381  }
13382 
13383  // Read global data:
13384  //------------------
13385 
13386  // Number of global data
13387  unsigned Nglobal = Global_data_pt.size();
13388 
13389  // Read line up to termination sign
13390  getline(restart_file, input_string, '#');
13391 
13392  // Ignore rest of line
13393  restart_file.ignore(80, '\n');
13394 
13395  // Check # of nodes:
13396  unsigned long check_nglobal = atoi(input_string.c_str());
13397 
13398 
13399  if (restart_file_is_open)
13400  {
13401  if (check_nglobal != Nglobal)
13402  {
13403  std::ostringstream error_message;
13404  error_message << "The number of global data " << Nglobal
13405  << " is not equal to that specified in the input file "
13406  << check_nglobal << std::endl;
13407 
13408  throw OomphLibError(error_message.str(),
13409  OOMPH_CURRENT_FUNCTION,
13410  OOMPH_EXCEPTION_LOCATION);
13411  }
13412  }
13413 
13414  for (unsigned iglobal = 0; iglobal < Nglobal; iglobal++)
13415  {
13416  Global_data_pt[iglobal]->read(restart_file);
13417  }
13418  }
13419 
13420  //===================================================================
13421  /// Set all timesteps to the same value, dt, and assign
13422  /// weights for all timesteppers in the problem.
13423  //===================================================================
13424  void Problem::initialise_dt(const double& dt)
13425  {
13426  // Initialise the timesteps in the Problem's time object
13427  Time_pt->initialise_dt(dt);
13428 
13429  // Find out how many timesteppers there are
13430  unsigned n_time_steppers = ntime_stepper();
13431 
13432  // Loop over them all and set the weights
13433  for (unsigned i = 0; i < n_time_steppers; i++)
13434  {
13436  if (time_stepper_pt(i)->adaptive_flag())
13437  {
13439  }
13440  }
13441  }
13442 
13443  //=========================================================================
13444  /// Set the value of the timesteps to be equal to the values passed in
13445  /// a vector and assign weights for all timesteppers in the problem
13446  //========================================================================
13448  {
13449  // Initialise the timesteps in the Problem's time object
13450  Time_pt->initialise_dt(dt);
13451 
13452  // Find out how many timesteppers there are
13453  unsigned n_time_steppers = ntime_stepper();
13454 
13455  // Loop over them all and set the weights
13456  for (unsigned i = 0; i < n_time_steppers; i++)
13457  {
13459  if (time_stepper_pt(i)->adaptive_flag())
13460  {
13462  }
13463  }
13464  }
13465 
13466  //========================================================
13467  /// Self-test: Check meshes and global data. Return 0 for OK
13468  //========================================================
13470  {
13471  // Initialise
13472  bool passed = true;
13473 
13474  // Are there any submeshes?
13475  unsigned Nmesh = nsub_mesh();
13476 
13477  // Just one mesh: Check it
13478  if (Nmesh == 0)
13479  {
13480  if (mesh_pt()->self_test() != 0)
13481  {
13482  passed = false;
13483  oomph_info
13484  << "\n ERROR: Failed Mesh::self_test() for single mesh in problem"
13485  << std::endl;
13486  }
13487  }
13488  // Loop over all submeshes and check them
13489  else
13490  {
13491  for (unsigned imesh = 0; imesh < Nmesh; imesh++)
13492  {
13493  if (mesh_pt(imesh)->self_test() != 0)
13494  {
13495  passed = false;
13496  oomph_info << "\n ERROR: Failed Mesh::self_test() for mesh imesh"
13497  << imesh << std::endl;
13498  }
13499  }
13500  }
13501 
13502 
13503  // Check global data
13504  unsigned Nglobal = Global_data_pt.size();
13505  for (unsigned iglobal = 0; iglobal < Nglobal; iglobal++)
13506  {
13507  if (Global_data_pt[iglobal]->self_test() != 0)
13508  {
13509  passed = false;
13510  oomph_info
13511  << "\n ERROR: Failed Data::self_test() for global data iglobal"
13512  << iglobal << std::endl;
13513  }
13514  }
13515 
13516 
13517 #ifdef OOMPH_HAS_MPI
13518 
13520  {
13521  // Note: This throws an error if it fails so no return is required.
13522  DocInfo tmp_doc_info;
13523  tmp_doc_info.disable_doc();
13524  check_halo_schemes(tmp_doc_info);
13525  }
13526 
13527 #endif
13528 
13529  // Return verdict
13530  if (passed)
13531  {
13532  return 0;
13533  }
13534  else
13535  {
13536  return 1;
13537  }
13538  }
13539 
13540  //====================================================================
13541  /// A function that is used to adapt a bifurcation-tracking
13542  /// problem, which requires separate interpolation of the
13543  /// associated eigenfunction. The error measure is chosen to be
13544  /// a suitable combination of the errors in the base flow and the
13545  /// eigenfunction. The bifurcation type is passed as an argument
13546  //=====================================================================
13547  void Problem::bifurcation_adapt_helper(unsigned& n_refined,
13548  unsigned& n_unrefined,
13549  const unsigned& bifurcation_type,
13550  const bool& actually_adapt)
13551  {
13552  // Storage for eigenfunction from the problem
13553  Vector<DoubleVector> eigenfunction;
13554  // Get the eigenfunction from the problem
13555  this->get_bifurcation_eigenfunction(eigenfunction);
13556 
13557  // Get the bifurcation parameter
13558  double* parameter_pt = this->bifurcation_parameter_pt();
13559 
13560  // Get the frequency parameter if tracking a Hopf bifurcation
13561  double omega = 0.0;
13562  // If we're tracking a Hopf then also get the frequency
13563  if (bifurcation_type == 3)
13564  {
13565  omega = dynamic_cast<HopfHandler*>(assembly_handler_pt())->omega();
13566  }
13567 
13568  // If we're tracking a Pitchfork get the slack parameter (Hack)
13569  double sigma = 0.0;
13570  if (bifurcation_type == 2)
13571  {
13572  sigma = this->dof(this->ndof() - 1);
13573  }
13574 
13575  // We can now deactivate the bifurcation tracking in the problem
13576  // to restore the degrees of freedom to the unaugmented value
13578 
13579  // Next, we create copies of the present problem
13580  // The number of copies depends on the number of eigenfunctions
13581  // One copy for each eigenfunction
13582  const unsigned n_copies = eigenfunction.size();
13583  Copy_of_problem_pt.resize(n_copies);
13584 
13585  // Loop over the number of copies
13586  for (unsigned c = 0; c < n_copies; c++)
13587  {
13588  // If we don't already have a copy
13589  if (Copy_of_problem_pt[c] == 0)
13590  {
13591  // Create the copy
13592  Copy_of_problem_pt[c] = this->make_copy();
13593 
13594  // Refine the copy to the same level as the current problem
13595 
13596  // Find number of submeshes
13597  const unsigned N_mesh = Copy_of_problem_pt[c]->nsub_mesh();
13598  // If there is only one mesh
13599  if (N_mesh == 0)
13600  {
13601  // Can we refine the mesh
13602  if (TreeBasedRefineableMeshBase* mmesh_pt =
13603  dynamic_cast<TreeBasedRefineableMeshBase*>(
13604  Copy_of_problem_pt[c]->mesh_pt(0)))
13605  {
13606  // Is the adapt flag set
13607  if (mmesh_pt->is_adaptation_enabled())
13608  {
13609  // Now get the original problem's mesh if it's refineable
13610  if (TreeBasedRefineableMeshBase* original_mesh_pt =
13611  dynamic_cast<TreeBasedRefineableMeshBase*>(
13612  this->mesh_pt(0)))
13613  {
13614  mmesh_pt->refine_base_mesh_as_in_reference_mesh(
13615  original_mesh_pt);
13616  }
13617  else
13618  {
13619  oomph_info
13620  << "Info/Warning: Mesh in orginal problem is not refineable."
13621  << std::endl;
13622  }
13623  }
13624  else
13625  {
13626  oomph_info << "Info/Warning: Mesh adaptation is disabled in copy."
13627  << std::endl;
13628  }
13629  }
13630  else
13631  {
13632  oomph_info << "Info/Warning: Mesh cannot be adapted in copy."
13633  << std::endl;
13634  }
13635  } // End of single mesh case
13636  // Otherwise loop over the submeshes
13637  else
13638  {
13639  for (unsigned m = 0; m < N_mesh; m++)
13640  {
13641  // Can we refine the submesh
13642  if (TreeBasedRefineableMeshBase* mmesh_pt =
13643  dynamic_cast<TreeBasedRefineableMeshBase*>(
13644  Copy_of_problem_pt[c]->mesh_pt(m)))
13645  {
13646  // Is the adapt flag set
13647  if (mmesh_pt->is_adaptation_enabled())
13648  {
13649  // Now get the original problem's mesh
13650  if (TreeBasedRefineableMeshBase* original_mesh_pt =
13651  dynamic_cast<TreeBasedRefineableMeshBase*>(
13652  this->mesh_pt(m)))
13653  {
13654  mmesh_pt->refine_base_mesh_as_in_reference_mesh(
13655  original_mesh_pt);
13656  }
13657  else
13658  {
13659  oomph_info << "Info/Warning: Mesh in orginal problem is not "
13660  "refineable."
13661  << std::endl;
13662  }
13663  }
13664  else
13665  {
13666  oomph_info
13667  << "Info/Warning: Mesh adaptation is disabled in copy."
13668  << std::endl;
13669  }
13670  }
13671  else
13672  {
13673  oomph_info << "Info/Warning: Mesh cannot be adapted in copy."
13674  << std::endl;
13675  }
13676  }
13677  // rebuild the global mesh in the copy
13678  Copy_of_problem_pt[c]->rebuild_global_mesh();
13679 
13680  } // End of multiple mesh case
13681 
13682  // Must call actions after adapt
13683  Copy_of_problem_pt[c]->actions_after_adapt();
13684 
13685  // Assign the equation numbers to the copy (quietly)
13687  }
13688  } // End of creation of copies
13689 
13690 
13691  // Now check some numbers
13692  for (unsigned c = 0; c < n_copies; c++)
13693  {
13694  // Check that the dofs match for each copy
13695 #ifdef PARANOID
13696  // If the problems don't match then complain
13697  if (Copy_of_problem_pt[c]->ndof() != this->ndof())
13698  {
13699  std::ostringstream error_stream;
13700  error_stream << "Number of unknowns in the problem copy " << c << " "
13701  << "not equal to number in the original:\n"
13702  << this->ndof() << " (original) "
13703  << Copy_of_problem_pt[c]->ndof() << " (copy)\n";
13704 
13705  throw OomphLibError(
13706  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
13707  }
13708 #endif
13709 
13710  // Assign the eigenfunction(s) to the copied problems
13711  Copy_of_problem_pt[c]->assign_eigenvector_to_dofs(eigenfunction[c]);
13712  // Set all pinned values to zero
13713  Copy_of_problem_pt[c]->set_pinned_values_to_zero();
13714  }
13715 
13716  // Symmetrise the problem if we are solving a pitchfork
13717  if (bifurcation_type == 2)
13718  {
13720  ->symmetrise_eigenfunction_for_adaptive_pitchfork_tracking();
13721  }
13722 
13723  // Find error estimates based on current problem and eigenproblem
13724  // Now we need to get the error estimates for both problems.
13725  Vector<Vector<double>> base_error, eigenfunction_error;
13726  this->get_all_error_estimates(base_error);
13727  // Loop over the copies
13728  for (unsigned c = 0; c < n_copies; c++)
13729  {
13730  // Get the error estimates for the copy
13731  Copy_of_problem_pt[c]->get_all_error_estimates(eigenfunction_error);
13732 
13733  // Find the number of meshes
13734  unsigned n_mesh = base_error.size();
13735 
13736 #ifdef PARANOID
13737  if (n_mesh != eigenfunction_error.size())
13738  {
13739  std::ostringstream error_stream;
13740  error_stream << "Problems do not have the same number of meshes\n"
13741  << "Base : " << n_mesh
13742  << " : Eigenproblem : " << eigenfunction_error.size()
13743  << "\n";
13744  throw OomphLibError(
13745  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
13746  }
13747 #endif
13748 
13749  for (unsigned m = 0; m < n_mesh; m++)
13750  {
13751  // Check the number of elements is the same
13752  unsigned n_element = base_error[m].size();
13753 #ifdef PARANOID
13754  if (n_element != eigenfunction_error[m].size())
13755  {
13756  std::ostringstream error_stream;
13757  error_stream << "Mesh " << m
13758  << " does not have the same number of elements in the "
13759  "two problems:\n"
13760  << "Base: " << n_element
13761  << " : Eigenproblem: " << eigenfunction_error[m].size()
13762  << "\n";
13763  throw OomphLibError(error_stream.str(),
13764  OOMPH_CURRENT_FUNCTION,
13765  OOMPH_EXCEPTION_LOCATION);
13766  }
13767 #endif
13768  // Now add all the error esimates together
13769  for (unsigned e = 0; e < n_element; e++)
13770  {
13771  // Add the error estimates (lazy)
13772  base_error[m][e] += eigenfunction_error[m][e];
13773  }
13774  }
13775  } // End of loop over copies
13776 
13777  // Then refine all problems based on the combined measure
13778  // if we are actually adapting (not just estimating the errors)
13779  if (actually_adapt)
13780  {
13781  this->adapt_based_on_error_estimates(n_refined, n_unrefined, base_error);
13782  for (unsigned c = 0; c < n_copies; c++)
13783  {
13784  Copy_of_problem_pt[c]->adapt_based_on_error_estimates(
13785  n_refined, n_unrefined, base_error);
13786  }
13787  // Symmetrise the problem (again) if we are solving for a pitchfork
13788  if (bifurcation_type == 2)
13789  {
13791  ->symmetrise_eigenfunction_for_adaptive_pitchfork_tracking();
13792  }
13793 
13794  // Now get the refined guess for the eigenvector
13795  for (unsigned c = 0; c < n_copies; c++)
13796  {
13797  Copy_of_problem_pt[c]->get_dofs(eigenfunction[c]);
13798  }
13799  }
13800 
13801  // Reactivate the tracking
13802  switch (bifurcation_type)
13803  {
13804  // Fold tracking
13805  case 1:
13806  this->activate_fold_tracking(parameter_pt);
13807  break;
13808 
13809  // Pitchfork
13810  case 2:
13811  this->activate_pitchfork_tracking(parameter_pt, eigenfunction[0]);
13812  // reset the slack parameter
13813  this->dof(this->ndof() - 1) = sigma;
13814  break;
13815 
13816  // Hopf
13817  case 3:
13818  this->activate_hopf_tracking(
13819  parameter_pt, omega, eigenfunction[0], eigenfunction[1]);
13820  break;
13821 
13822  default:
13823  std::ostringstream error_stream;
13824  error_stream << "Bifurcation type " << bifurcation_type
13825  << " not known\n"
13826  << "1: Fold, 2: Pitchfork, 3: Hopf\n";
13827  throw OomphLibError(
13828  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
13829  }
13830  }
13831 
13832 
13833  //====================================================================
13834  /// A function that is used to document the errors when
13835  /// adapting a bifurcation-tracking
13836  /// problem, which requires separate interpolation of the
13837  /// associated eigenfunction. The error measure is chosen to be
13838  /// a suitable combination of the errors in the base flow and the
13839  /// eigenfunction. The bifurcation type is passed as an argument
13840  //=====================================================================
13841  void Problem::bifurcation_adapt_doc_errors(const unsigned& bifurcation_type)
13842  {
13843  // Dummy arguments
13844  unsigned n_refined, n_unrefined;
13845  // Just call the bifurcation helper without actually adapting
13846  bifurcation_adapt_helper(n_refined, n_unrefined, bifurcation_type, false);
13847  }
13848 
13849 
13850  //========================================================================
13851  /// Adapt problem:
13852  /// Perform mesh adaptation for (all) refineable (sub)mesh(es),
13853  /// based on their own error estimates and the target errors specified
13854  /// in the mesh(es). Following mesh adaptation,
13855  /// update global mesh, and re-assign equation numbers.
13856  /// Return # of refined/unrefined elements. On return from this
13857  /// function, Problem can immediately be solved again.
13858  //======================================================================
13859  void Problem::adapt(unsigned& n_refined, unsigned& n_unrefined)
13860  {
13861  double t_start_total = 0.0;
13863  {
13864  t_start_total = TimingHelpers::timer();
13865  }
13866 
13867  // Get the bifurcation type
13868  int bifurcation_type = this->Assembly_handler_pt->bifurcation_type();
13869 
13870  bool continuation_problem = false;
13871 
13872  // If we have continuation data then we need to project that across to the
13873  // new mesh
13875  {
13876  if (Dof_derivative.size() != 0)
13877  {
13878  continuation_problem = true;
13879  }
13880  }
13881 
13882  // If we are tracking a bifurcation then call the bifurcation adapt function
13883  if (bifurcation_type != 0)
13884  {
13885  this->bifurcation_adapt_helper(n_refined, n_unrefined, bifurcation_type);
13886  // Return immediately
13887  return;
13888  }
13889 
13890  if (continuation_problem)
13891  {
13892  // Create a copy of the problem
13893  Copy_of_problem_pt.resize(2);
13894  // If we don't already have a copy
13895  for (unsigned c = 0; c < 2; c++)
13896  {
13897  if (Copy_of_problem_pt[c] == 0)
13898  {
13899  // Create the copy
13900  Copy_of_problem_pt[c] = this->make_copy();
13901 
13902  // Refine the copy to the same level as the current problem
13903  // Must call actions before adapt
13904  Copy_of_problem_pt[c]->actions_before_adapt();
13905 
13906  // Find number of submeshes
13907  const unsigned N_mesh = Copy_of_problem_pt[c]->nsub_mesh();
13908 
13909  // If there is only one mesh
13910  if (N_mesh == 0)
13911  {
13912  // Can we refine the mesh
13913  if (TreeBasedRefineableMeshBase* mmesh_pt =
13914  dynamic_cast<TreeBasedRefineableMeshBase*>(
13915  Copy_of_problem_pt[c]->mesh_pt(0)))
13916  {
13917  // Is the adapt flag set
13918  if (mmesh_pt->is_adaptation_enabled())
13919  {
13920  // Now get the original problem's mesh if it's refineable
13921  if (TreeBasedRefineableMeshBase* original_mesh_pt =
13922  dynamic_cast<TreeBasedRefineableMeshBase*>(
13923  this->mesh_pt(0)))
13924  {
13925  if (dynamic_cast<SolidMesh*>(original_mesh_pt) != 0)
13926  {
13927  oomph_info
13928  << "Info/Warning: Adaptive Continuation is broken in "
13929  << "SolidElement" << std::endl;
13930  }
13931  mmesh_pt->refine_base_mesh_as_in_reference_mesh(
13932  original_mesh_pt);
13933  }
13934  else
13935  {
13936  oomph_info << "Info/Warning: Mesh in orginal problem is not "
13937  "refineable."
13938  << std::endl;
13939  }
13940  }
13941  else
13942  {
13943  oomph_info
13944  << "Info/Warning: Mesh adaptation is disabled in copy."
13945  << std::endl;
13946  }
13947  }
13948  else if (TriangleMeshBase* tmesh_pt =
13949  dynamic_cast<TriangleMeshBase*>(
13950  Copy_of_problem_pt[c]->mesh_pt(0)))
13951  {
13952  if (TriangleMeshBase* original_mesh_pt =
13953  dynamic_cast<TriangleMeshBase*>(this->mesh_pt(0)))
13954  {
13955  if (dynamic_cast<SolidMesh*>(original_mesh_pt) != 0)
13956  {
13957  oomph_info
13958  << "Info/Warning: Adaptive Continuation is broken in "
13959  << "SolidElement" << std::endl;
13960  }
13961 
13962  // Remesh using the triangulateIO of the base mesh
13963  // Done via a file, so a bit hacky but this will be
13964  // superseded very soon
13965  std::ofstream tri_dump("triangle_mesh.dmp");
13966  original_mesh_pt->dump_triangulateio(tri_dump);
13967  tri_dump.close();
13968  std::ifstream tri_read("triangle_mesh.dmp");
13969  tmesh_pt->remesh_from_triangulateio(tri_read);
13970  tri_read.close();
13971 
13972 
13973  // Set the nodes to be at the same positions
13974  // as the original just in case the
13975  // triangulatio is out of sync with the real data
13976  const unsigned n_node = original_mesh_pt->nnode();
13977  for (unsigned n = 0; n < n_node; ++n)
13978  {
13979  Node* const nod_pt = original_mesh_pt->node_pt(n);
13980  Node* const new_node_pt = tmesh_pt->node_pt(n);
13981  unsigned n_dim = nod_pt->ndim();
13982  for (unsigned i = 0; i < n_dim; ++i)
13983  {
13984  new_node_pt->x(i) = nod_pt->x(i);
13985  }
13986  }
13987  }
13988  else
13989  {
13990  oomph_info
13991  << "Info/warning: Original Mesh is not TriangleBased\n"
13992  << "... but the copy is!" << std::endl;
13993  }
13994  }
13995  else
13996  {
13997  oomph_info << "Info/Warning: Mesh cannot be adapted in copy."
13998  << std::endl;
13999  }
14000  } // End of single mesh case
14001  // Otherwise loop over the submeshes
14002  else
14003  {
14004  for (unsigned m = 0; m < N_mesh; m++)
14005  {
14006  // Can we refine the submesh
14007  if (TreeBasedRefineableMeshBase* mmesh_pt =
14008  dynamic_cast<TreeBasedRefineableMeshBase*>(
14009  Copy_of_problem_pt[c]->mesh_pt(m)))
14010  {
14011  // Is the adapt flag set
14012  if (mmesh_pt->is_adaptation_enabled())
14013  {
14014  // Now get the original problem's mesh
14015  if (TreeBasedRefineableMeshBase* original_mesh_pt =
14016  dynamic_cast<TreeBasedRefineableMeshBase*>(
14017  this->mesh_pt(m)))
14018  {
14019  if (dynamic_cast<SolidMesh*>(original_mesh_pt) != 0)
14020  {
14021  oomph_info
14022  << "Info/Warning: Adaptive Continuation is broken in "
14023  << "SolidElement" << std::endl;
14024  }
14025 
14026  mmesh_pt->refine_base_mesh_as_in_reference_mesh(
14027  original_mesh_pt);
14028  }
14029  else
14030  {
14031  oomph_info << "Info/Warning: Mesh in orginal problem is "
14032  "not refineable."
14033  << std::endl;
14034  }
14035  }
14036  else
14037  {
14038  oomph_info
14039  << "Info/Warning: Mesh adaptation is disabled in copy."
14040  << std::endl;
14041  }
14042  }
14043  else if (TriangleMeshBase* tmesh_pt =
14044  dynamic_cast<TriangleMeshBase*>(
14045  Copy_of_problem_pt[c]->mesh_pt(m)))
14046  {
14047  if (TriangleMeshBase* original_mesh_pt =
14048  dynamic_cast<TriangleMeshBase*>(this->mesh_pt(m)))
14049  {
14050  if (dynamic_cast<SolidMesh*>(original_mesh_pt) != 0)
14051  {
14052  oomph_info
14053  << "Info/Warning: Adaptive Continuation is broken in "
14054  << "SolidElement" << std::endl;
14055  }
14056 
14057  // Remesh using the triangulateIO of the base mesh
14058  // Done via a file, so a bit hacky but this will be
14059  // superseded very soon
14060  std::ofstream tri_dump("triangle_mesh.dmp");
14061  original_mesh_pt->dump_triangulateio(tri_dump);
14062  tri_dump.close();
14063  std::ifstream tri_read("triangle_mesh.dmp");
14064  tmesh_pt->remesh_from_triangulateio(tri_read);
14065  tri_read.close();
14066 
14067  // Set the nodes to be at the same positions
14068  // as the original just in case the
14069  // triangulatio is out of sync with the real data
14070  const unsigned n_node = original_mesh_pt->nnode();
14071  for (unsigned n = 0; n < n_node; ++n)
14072  {
14073  Node* const nod_pt = original_mesh_pt->node_pt(n);
14074  Node* const new_node_pt = tmesh_pt->node_pt(n);
14075  unsigned n_dim = nod_pt->ndim();
14076  for (unsigned i = 0; i < n_dim; ++i)
14077  {
14078  new_node_pt->x(i) = nod_pt->x(i);
14079  }
14080  }
14081  }
14082  else
14083  {
14084  oomph_info
14085  << "Info/warning: Original Mesh is not TriangleBased\n"
14086  << "... but the copy is!" << std::endl;
14087  }
14088  }
14089  else
14090  {
14091  oomph_info << "Info/Warning: Mesh cannot be adapted in copy."
14092  << std::endl;
14093  }
14094  }
14095 
14096 
14097  // Must call actions after adapt
14098  Copy_of_problem_pt[c]->actions_after_adapt();
14099 
14100  // rebuild the global mesh in the copy
14101  Copy_of_problem_pt[c]->rebuild_global_mesh();
14102 
14103  } // End of multiple mesh case
14104 
14105  // Must call actions after adapt
14106  Copy_of_problem_pt[c]->actions_after_adapt();
14107 
14108  // Assign the equation numbers to the copy (quietly)
14110  }
14111 
14112  // Check that the dofs match for each copy
14113 #ifdef PARANOID
14114  // If the problems don't match then complain
14115  if (Copy_of_problem_pt[c]->ndof() != this->ndof())
14116  {
14117  std::ostringstream error_stream;
14118  error_stream << "Number of unknowns in the problem copy " << c << " "
14119  << "not equal to number in the original:\n"
14120  << this->ndof() << " (original) "
14121  << Copy_of_problem_pt[c]->ndof() << " (copy)\n";
14122 
14123  throw OomphLibError(error_stream.str(),
14124  OOMPH_CURRENT_FUNCTION,
14125  OOMPH_EXCEPTION_LOCATION);
14126  }
14127 #endif
14128  }
14129 
14130  // Need to set the Dof derivatives to the copied problem
14131  // Assign the eigenfunction(s) to the copied problems
14132  unsigned ndof_local = Dof_distribution_pt->nrow_local();
14133  for (unsigned i = 0; i < ndof_local; i++)
14134  {
14135  Copy_of_problem_pt[0]->dof(i) = this->dof_derivative(i);
14136  Copy_of_problem_pt[1]->dof(i) = this->dof_current(i);
14137  }
14138  // Set all pinned values to zero
14139  Copy_of_problem_pt[0]->set_pinned_values_to_zero();
14140  // Don't need to for the current dofs that are actuall the dofs
14141 
14142  // Now adapt
14143  Vector<Vector<double>> base_error;
14144  this->get_all_error_estimates(base_error);
14145  this->adapt_based_on_error_estimates(n_refined, n_unrefined, base_error);
14146  Copy_of_problem_pt[0]->adapt_based_on_error_estimates(
14147  n_refined, n_unrefined, base_error);
14148  Copy_of_problem_pt[1]->adapt_based_on_error_estimates(
14149  n_refined, n_unrefined, base_error);
14150 
14151  // Now sort out the Dof pointer
14152  ndof_local = Dof_distribution_pt->nrow_local();
14153  if (Dof_derivative.size() != ndof_local)
14154  {
14155  Dof_derivative.resize(ndof_local, 0.0);
14156  }
14157  if (Dof_current.size() != ndof_local)
14158  {
14159  Dof_current.resize(ndof_local, 0.0);
14160  }
14161  for (unsigned i = 0; i < ndof_local; i++)
14162  {
14163  Dof_derivative[i] = Copy_of_problem_pt[0]->dof(i);
14164  Dof_current[i] = Copy_of_problem_pt[1]->dof(i);
14165  }
14166  // Return immediately
14167  return;
14168  }
14169 
14170  oomph_info << std::endl << std::endl;
14171  oomph_info << "Adapting problem:" << std::endl;
14172  oomph_info << "=================" << std::endl;
14173 
14174  double t_start = 0.0;
14176  {
14177  t_start = TimingHelpers::timer();
14178  }
14179 
14180  // Call the actions before adaptation
14182 
14183  double t_end = 0.0;
14185  {
14186  t_end = TimingHelpers::timer();
14187  oomph_info << "Time for actions before adapt: " << t_end - t_start
14188  << std::endl;
14189  t_start = TimingHelpers::timer();
14190  }
14191 
14192  // Initialise counters
14193  n_refined = 0;
14194  n_unrefined = 0;
14195 
14196  // Number of submeshes?
14197  unsigned Nmesh = nsub_mesh();
14198 
14199  // Single mesh:
14200  //------------
14201  if (Nmesh == 0)
14202  {
14203  // Refine single mesh if possible
14204  if (RefineableMeshBase* mmesh_pt =
14205  dynamic_cast<RefineableMeshBase*>(mesh_pt(0)))
14206  {
14207  if (mmesh_pt->is_adaptation_enabled())
14208  {
14209  double t_start = TimingHelpers::timer();
14210 
14211  // Get pointer to error estimator
14212  ErrorEstimator* error_estimator_pt =
14213  mmesh_pt->spatial_error_estimator_pt();
14214 
14215 #ifdef PARANOID
14216  if (error_estimator_pt == 0)
14217  {
14218  throw OomphLibError("Error estimator hasn't been set yet",
14219  OOMPH_CURRENT_FUNCTION,
14220  OOMPH_EXCEPTION_LOCATION);
14221  }
14222 #endif
14223 
14224  // Get error for all elements
14225  Vector<double> elemental_error(mmesh_pt->nelement());
14226 
14227  if (mmesh_pt->doc_info_pt() == 0)
14228  {
14229  error_estimator_pt->get_element_errors(mesh_pt(0), elemental_error);
14230  }
14231  else
14232  {
14233  error_estimator_pt->get_element_errors(
14234  mesh_pt(0), elemental_error, *mmesh_pt->doc_info_pt());
14235  }
14236 
14237  // Store max./min actual error
14238  mmesh_pt->max_error() = std::fabs(*std::max_element(
14239  elemental_error.begin(), elemental_error.end(), AbsCmp<double>()));
14240 
14241  mmesh_pt->min_error() = std::fabs(*std::min_element(
14242  elemental_error.begin(), elemental_error.end(), AbsCmp<double>()));
14243 
14244  oomph_info << "\n Max/min error: " << mmesh_pt->max_error() << " "
14245  << mmesh_pt->min_error() << std::endl
14246  << std::endl;
14247 
14248 
14250  {
14251  t_end = TimingHelpers::timer();
14252  oomph_info << "Time for error estimation: " << t_end - t_start
14253  << std::endl;
14254  t_start = TimingHelpers::timer();
14255  }
14256 
14257  // Adapt mesh
14258  mmesh_pt->adapt(elemental_error);
14259 
14260  // Add to counters
14261  n_refined += mmesh_pt->nrefined();
14262  n_unrefined += mmesh_pt->nunrefined();
14263 
14265  {
14266  t_end = TimingHelpers::timer();
14267  oomph_info << "Time for complete mesh adaptation "
14268  << "(but excluding comp of error estimate): "
14269  << t_end - t_start << std::endl;
14270  t_start = TimingHelpers::timer();
14271  }
14272  }
14273  else
14274  {
14275  oomph_info << "Info/Warning: Mesh adaptation is disabled."
14276  << std::endl;
14277  }
14278  }
14279  else
14280  {
14281  oomph_info << "Info/Warning: Mesh cannot be adapted" << std::endl;
14282  }
14283  }
14284  // Multiple submeshes
14285  //------------------
14286  else
14287  {
14288  // Loop over submeshes
14289  for (unsigned imesh = 0; imesh < Nmesh; imesh++)
14290  {
14291  // Refine single mesh uniformly if possible
14292  if (RefineableMeshBase* mmesh_pt =
14293  dynamic_cast<RefineableMeshBase*>(mesh_pt(imesh)))
14294  {
14295  double t_start = TimingHelpers::timer();
14296 
14297  // Get pointer to error estimator
14298  ErrorEstimator* error_estimator_pt =
14299  mmesh_pt->spatial_error_estimator_pt();
14300 
14301 #ifdef PARANOID
14302  if (error_estimator_pt == 0)
14303  {
14304  throw OomphLibError("Error estimator hasn't been set yet",
14305  OOMPH_CURRENT_FUNCTION,
14306  OOMPH_EXCEPTION_LOCATION);
14307  }
14308 #endif
14309 
14310  if (mmesh_pt->is_adaptation_enabled())
14311  {
14312  // Get error for all elements
14313  Vector<double> elemental_error(mmesh_pt->nelement());
14314  if (mmesh_pt->doc_info_pt() == 0)
14315  {
14316  error_estimator_pt->get_element_errors(mesh_pt(imesh),
14317  elemental_error);
14318  }
14319  else
14320  {
14321  error_estimator_pt->get_element_errors(
14322  mesh_pt(imesh), elemental_error, *mmesh_pt->doc_info_pt());
14323  }
14324 
14325  // Store max./min error if the mesh has any elements
14326  if (mesh_pt(imesh)->nelement() > 0)
14327  {
14328  mmesh_pt->max_error() =
14329  std::fabs(*std::max_element(elemental_error.begin(),
14330  elemental_error.end(),
14331  AbsCmp<double>()));
14332 
14333  mmesh_pt->min_error() =
14334  std::fabs(*std::min_element(elemental_error.begin(),
14335  elemental_error.end(),
14336  AbsCmp<double>()));
14337  }
14338 
14339  oomph_info << "\n Max/min error: " << mmesh_pt->max_error() << " "
14340  << mmesh_pt->min_error() << std::endl;
14341 
14342 
14344  {
14345  t_end = TimingHelpers::timer();
14346  oomph_info << "Time for error estimation: " << t_end - t_start
14347  << std::endl;
14348  t_start = TimingHelpers::timer();
14349  }
14350 
14351  // Adapt mesh
14352  mmesh_pt->adapt(elemental_error);
14353 
14354  // Add to counters
14355  n_refined += mmesh_pt->nrefined();
14356  n_unrefined += mmesh_pt->nunrefined();
14357 
14358 
14360  {
14361  t_end = TimingHelpers::timer();
14362  oomph_info << "Time for complete mesh adaptation "
14363  << "(but excluding comp of error estimate): "
14364  << t_end - t_start << std::endl;
14365  t_start = TimingHelpers::timer();
14366  }
14367  }
14368  else
14369  {
14370  oomph_info << "Info/Warning: Mesh adaptation is disabled."
14371  << std::endl;
14372  }
14373  }
14374  else
14375  {
14376  oomph_info << "Info/Warning: Mesh cannot be adapted." << std::endl;
14377  }
14378 
14379  } // End of loop over submeshes
14380 
14381  // Rebuild the global mesh
14383  }
14384 
14385 
14387  {
14388  t_end = TimingHelpers::timer();
14389  oomph_info << "Total time for actual adaptation "
14390  << "(all meshes; incl error estimates): " << t_end - t_start
14391  << std::endl;
14392  t_start = TimingHelpers::timer();
14393  }
14394 
14395  // Any actions after adapt
14397 
14398 
14400  {
14401  t_end = TimingHelpers::timer();
14402  oomph_info << "Time for actions after adapt: " << t_end - t_start
14403  << std::endl;
14404  t_start = TimingHelpers::timer();
14405 
14406  oomph_info << "About to start re-assigning eqn numbers "
14407  << "with Problem::assign_eqn_numbers() at end of "
14408  << "Problem::adapt().\n";
14409  }
14410 
14411  // Attach the boundary conditions to the mesh
14412  oomph_info << "\nNumber of equations: " << assign_eqn_numbers() << std::endl
14413  << std::endl;
14414 
14415 
14417  {
14418  t_end = TimingHelpers::timer();
14419  oomph_info << "Time for re-assigning eqn numbers with "
14420  << "Problem::assign_eqn_numbers() at end of Problem::adapt(): "
14421  << t_end - t_start << std::endl;
14422  oomph_info << "Total time for adapt: " << t_end - t_start_total
14423  << std::endl;
14424  }
14425  }
14426 
14427  //========================================================================
14428  /// p-adapt problem:
14429  /// Perform mesh adaptation for (all) refineable (sub)mesh(es),
14430  /// based on their own error estimates and the target errors specified
14431  /// in the mesh(es). Following mesh adaptation,
14432  /// update global mesh, and re-assign equation numbers.
14433  /// Return # of refined/unrefined elements. On return from this
14434  /// function, Problem can immediately be solved again.
14435  //======================================================================
14436  void Problem::p_adapt(unsigned& n_refined, unsigned& n_unrefined)
14437  {
14438  double t_start_total = 0.0;
14440  {
14441  t_start_total = TimingHelpers::timer();
14442  }
14443 
14444  // Get the bifurcation type
14445  int bifurcation_type = this->Assembly_handler_pt->bifurcation_type();
14446 
14447  // If we are tracking a bifurcation then call the bifurcation adapt function
14448  if (bifurcation_type != 0)
14449  {
14450  this->bifurcation_adapt_helper(n_refined, n_unrefined, bifurcation_type);
14451  // Return immediately
14452  return;
14453  }
14454 
14455  oomph_info << std::endl << std::endl;
14456  oomph_info << "p-adapting problem:" << std::endl;
14457  oomph_info << "===================" << std::endl;
14458 
14459  double t_start = 0.0;
14461  {
14462  t_start = TimingHelpers::timer();
14463  }
14464 
14465  // Call the actions before adaptation
14467 
14468  double t_end = 0.0;
14470  {
14471  t_end = TimingHelpers::timer();
14472  oomph_info << "Time for actions before adapt: " << t_end - t_start
14473  << std::endl;
14474  t_start = TimingHelpers::timer();
14475  }
14476 
14477  // Initialise counters
14478  n_refined = 0;
14479  n_unrefined = 0;
14480 
14481  // Number of submeshes?
14482  unsigned Nmesh = nsub_mesh();
14483 
14484  // Single mesh:
14485  //------------
14486  if (Nmesh == 0)
14487  {
14488  // Refine single mesh if possible
14489  if (RefineableMeshBase* mmesh_pt =
14490  dynamic_cast<RefineableMeshBase*>(mesh_pt(0)))
14491  {
14492  if (mmesh_pt->is_p_adaptation_enabled())
14493  {
14494  double t_start = TimingHelpers::timer();
14495 
14496  // Get pointer to error estimator
14497  ErrorEstimator* error_estimator_pt =
14498  mmesh_pt->spatial_error_estimator_pt();
14499 
14500 #ifdef PARANOID
14501  if (error_estimator_pt == 0)
14502  {
14503  throw OomphLibError("Error estimator hasn't been set yet",
14504  OOMPH_CURRENT_FUNCTION,
14505  OOMPH_EXCEPTION_LOCATION);
14506  }
14507 #endif
14508 
14509  // Get error for all elements
14510  Vector<double> elemental_error(mmesh_pt->nelement());
14511 
14512  if (mmesh_pt->doc_info_pt() == 0)
14513  {
14514  error_estimator_pt->get_element_errors(mesh_pt(0), elemental_error);
14515  }
14516  else
14517  {
14518  error_estimator_pt->get_element_errors(
14519  mesh_pt(0), elemental_error, *mmesh_pt->doc_info_pt());
14520  }
14521 
14522  // Store max./min actual error
14523  mmesh_pt->max_error() = std::fabs(*std::max_element(
14524  elemental_error.begin(), elemental_error.end(), AbsCmp<double>()));
14525 
14526  mmesh_pt->min_error() = std::fabs(*std::min_element(
14527  elemental_error.begin(), elemental_error.end(), AbsCmp<double>()));
14528 
14529  oomph_info << "\n Max/min error: " << mmesh_pt->max_error() << " "
14530  << mmesh_pt->min_error() << std::endl
14531  << std::endl;
14532 
14533 
14535  {
14536  t_end = TimingHelpers::timer();
14537  oomph_info << "Time for error estimation: " << t_end - t_start
14538  << std::endl;
14539  t_start = TimingHelpers::timer();
14540  }
14541 
14542  // Adapt mesh
14543  mmesh_pt->p_adapt(elemental_error);
14544 
14545  // Add to counters
14546  n_refined += mmesh_pt->nrefined();
14547  n_unrefined += mmesh_pt->nunrefined();
14548 
14550  {
14551  t_end = TimingHelpers::timer();
14552  oomph_info << "Time for complete mesh adaptation "
14553  << "(but excluding comp of error estimate): "
14554  << t_end - t_start << std::endl;
14555  t_start = TimingHelpers::timer();
14556  }
14557  }
14558  else
14559  {
14560  oomph_info << "Info/Warning: Mesh adaptation is disabled."
14561  << std::endl;
14562  }
14563  }
14564  else
14565  {
14566  oomph_info << "Info/Warning: Mesh cannot be adapted" << std::endl;
14567  }
14568  }
14569  // Multiple submeshes
14570  //------------------
14571  else
14572  {
14573  // Loop over submeshes
14574  for (unsigned imesh = 0; imesh < Nmesh; imesh++)
14575  {
14576  // Refine single mesh uniformly if possible
14577  if (RefineableMeshBase* mmesh_pt =
14578  dynamic_cast<RefineableMeshBase*>(mesh_pt(imesh)))
14579  {
14580  double t_start = TimingHelpers::timer();
14581 
14582  // Get pointer to error estimator
14583  ErrorEstimator* error_estimator_pt =
14584  mmesh_pt->spatial_error_estimator_pt();
14585 
14586 #ifdef PARANOID
14587  if (error_estimator_pt == 0)
14588  {
14589  throw OomphLibError("Error estimator hasn't been set yet",
14590  OOMPH_CURRENT_FUNCTION,
14591  OOMPH_EXCEPTION_LOCATION);
14592  }
14593 #endif
14594 
14595  if (mmesh_pt->is_p_adaptation_enabled())
14596  {
14597  // Get error for all elements
14598  Vector<double> elemental_error(mmesh_pt->nelement());
14599  if (mmesh_pt->doc_info_pt() == 0)
14600  {
14601  error_estimator_pt->get_element_errors(mesh_pt(imesh),
14602  elemental_error);
14603  }
14604  else
14605  {
14606  error_estimator_pt->get_element_errors(
14607  mesh_pt(imesh), elemental_error, *mmesh_pt->doc_info_pt());
14608  }
14609 
14610  // Store max./min error if the mesh has any elements
14611  if (mesh_pt(imesh)->nelement() > 0)
14612  {
14613  mmesh_pt->max_error() =
14614  std::fabs(*std::max_element(elemental_error.begin(),
14615  elemental_error.end(),
14616  AbsCmp<double>()));
14617 
14618  mmesh_pt->min_error() =
14619  std::fabs(*std::min_element(elemental_error.begin(),
14620  elemental_error.end(),
14621  AbsCmp<double>()));
14622  }
14623 
14624  oomph_info << "\n Max/min error: " << mmesh_pt->max_error() << " "
14625  << mmesh_pt->min_error() << std::endl;
14626 
14627 
14629  {
14630  t_end = TimingHelpers::timer();
14631  oomph_info << "Time for error estimation: " << t_end - t_start
14632  << std::endl;
14633  t_start = TimingHelpers::timer();
14634  }
14635 
14636  // Adapt mesh
14637  mmesh_pt->p_adapt(elemental_error);
14638 
14639  // Add to counters
14640  n_refined += mmesh_pt->nrefined();
14641  n_unrefined += mmesh_pt->nunrefined();
14642 
14643 
14645  {
14646  t_end = TimingHelpers::timer();
14647  oomph_info << "Time for complete mesh adaptation "
14648  << "(but excluding comp of error estimate): "
14649  << t_end - t_start << std::endl;
14650  t_start = TimingHelpers::timer();
14651  }
14652  }
14653  else
14654  {
14655  oomph_info << "Info/Warning: Mesh adaptation is disabled."
14656  << std::endl;
14657  }
14658  }
14659  else
14660  {
14661  oomph_info << "Info/Warning: Mesh cannot be adapted." << std::endl;
14662  }
14663 
14664  } // End of loop over submeshes
14665 
14666  // Rebuild the global mesh
14668  }
14669 
14670 
14672  {
14673  t_end = TimingHelpers::timer();
14674  oomph_info << "Total time for actual adaptation "
14675  << "(all meshes; incl error estimates): " << t_end - t_start
14676  << std::endl;
14677  t_start = TimingHelpers::timer();
14678  }
14679 
14680  // Any actions after adapt
14682 
14683 
14685  {
14686  t_end = TimingHelpers::timer();
14687  oomph_info << "Time for actions after adapt: " << t_end - t_start
14688  << std::endl;
14689  t_start = TimingHelpers::timer();
14690 
14691  oomph_info << "About to start re-assigning eqn numbers "
14692  << "with Problem::assign_eqn_numbers() at end of "
14693  << "Problem::adapt().\n";
14694  }
14695 
14696  // Attach the boundary conditions to the mesh
14697  oomph_info << "\nNumber of equations: " << assign_eqn_numbers() << std::endl
14698  << std::endl;
14699 
14700 
14702  {
14703  t_end = TimingHelpers::timer();
14704  oomph_info << "Time for re-assigning eqn numbers with "
14705  << "Problem::assign_eqn_numbers() at end of Problem::adapt(): "
14706  << t_end - t_start << std::endl;
14707  oomph_info << "Total time for adapt: " << t_end - t_start_total
14708  << std::endl;
14709  }
14710  }
14711 
14712  //========================================================================
14713  /// Perform mesh adaptation for (all) refineable (sub)mesh(es),
14714  /// based on the error estimates in elemental_error
14715  /// and the target errors specified
14716  /// in the mesh(es). Following mesh adaptation,
14717  /// update global mesh, and re-assign equation numbers.
14718  /// Return # of refined/unrefined elements. On return from this
14719  /// function, Problem can immediately be solved again.
14720  //========================================================================
14722  unsigned& n_refined,
14723  unsigned& n_unrefined,
14724  Vector<Vector<double>>& elemental_error)
14725  {
14726  oomph_info << std::endl << std::endl;
14727  oomph_info << "Adapting problem:" << std::endl;
14728  oomph_info << "=================" << std::endl;
14729 
14730  // Call the actions before adaptation
14732 
14733  // Initialise counters
14734  n_refined = 0;
14735  n_unrefined = 0;
14736 
14737  // Number of submeshes?
14738  unsigned Nmesh = nsub_mesh();
14739 
14740  // Single mesh:
14741  //------------
14742  if (Nmesh == 0)
14743  {
14744  // Refine single mesh uniformly if possible
14745  if (RefineableMeshBase* mmesh_pt =
14746  dynamic_cast<RefineableMeshBase*>(Problem::mesh_pt(0)))
14747  {
14748  if (mmesh_pt->is_adaptation_enabled())
14749  {
14750  // Adapt mesh
14751  mmesh_pt->adapt(elemental_error[0]);
14752 
14753  // Add to counters
14754  n_refined += mmesh_pt->nrefined();
14755  n_unrefined += mmesh_pt->nunrefined();
14756  }
14757  else
14758  {
14759  oomph_info << "Info/Warning: Mesh adaptation is disabled."
14760  << std::endl;
14761  }
14762  }
14763  else
14764  {
14765  oomph_info << "Info/Warning: Mesh cannot be adapted" << std::endl;
14766  }
14767  }
14768 
14769  // Multiple submeshes
14770  //------------------
14771  else
14772  {
14773  // Loop over submeshes
14774  for (unsigned imesh = 0; imesh < Nmesh; imesh++)
14775  {
14776  // Refine single mesh uniformly if possible
14777  if (RefineableMeshBase* mmesh_pt =
14778  dynamic_cast<RefineableMeshBase*>(Problem::mesh_pt(imesh)))
14779  {
14780  if (mmesh_pt->is_adaptation_enabled())
14781  {
14782  // Adapt mesh
14783  mmesh_pt->adapt(elemental_error[imesh]);
14784 
14785  // Add to counters
14786  n_refined += mmesh_pt->nrefined();
14787  n_unrefined += mmesh_pt->nunrefined();
14788  }
14789  else
14790  {
14791  oomph_info << "Info/Warning: Mesh adaptation is disabled."
14792  << std::endl;
14793  }
14794  }
14795  else
14796  {
14797  oomph_info << "Info/Warning: Mesh cannot be adapted." << std::endl;
14798  }
14799 
14800  } // End of loop over submeshes
14801 
14802  // Rebuild the global mesh
14804  }
14805 
14806  // Any actions after adapt
14808 
14809  // Attach the boundary conditions to the mesh
14810  oomph_info << "\nNumber of equations: " << assign_eqn_numbers() << std::endl
14811  << std::endl;
14812  }
14813 
14814 
14815  //========================================================================
14816  /// Return the error estimates computed by (all) refineable
14817  /// (sub)mesh(es) in the elemental_error structure, which consists of
14818  /// a vector of elemental errors for each (sub)mesh.
14819  //========================================================================
14821  {
14822  // Number of submeshes?
14823  const unsigned Nmesh = nsub_mesh();
14824 
14825  // Single mesh:
14826  //------------
14827  if (Nmesh == 0)
14828  {
14829  // There is only one mesh
14830  elemental_error.resize(1);
14831  // Refine single mesh uniformly if possible
14832  if (RefineableMeshBase* mmesh_pt =
14833  dynamic_cast<RefineableMeshBase*>(Problem::mesh_pt(0)))
14834  {
14835  // If we can adapt the mesh
14836  if (mmesh_pt->is_adaptation_enabled())
14837  {
14838  // Get pointer to error estimator
14839  ErrorEstimator* error_estimator_pt =
14840  mmesh_pt->spatial_error_estimator_pt();
14841 
14842 #ifdef PARANOID
14843  if (error_estimator_pt == 0)
14844  {
14845  throw OomphLibError("Error estimator hasn't been set yet",
14846  OOMPH_CURRENT_FUNCTION,
14847  OOMPH_EXCEPTION_LOCATION);
14848  }
14849 #endif
14850 
14851  // Get error for all elements
14852  elemental_error[0].resize(mmesh_pt->nelement());
14853  // Are we documenting the errors or not
14854  if (mmesh_pt->doc_info_pt() == 0)
14855  {
14856  error_estimator_pt->get_element_errors(Problem::mesh_pt(0),
14857  elemental_error[0]);
14858  }
14859  else
14860  {
14861  error_estimator_pt->get_element_errors(Problem::mesh_pt(0),
14862  elemental_error[0],
14863  *mmesh_pt->doc_info_pt());
14864  }
14865 
14866  // Store max./min actual error
14867  mmesh_pt->max_error() =
14868  std::fabs(*std::max_element(elemental_error[0].begin(),
14869  elemental_error[0].end(),
14870  AbsCmp<double>()));
14871 
14872  mmesh_pt->min_error() =
14873  std::fabs(*std::min_element(elemental_error[0].begin(),
14874  elemental_error[0].end(),
14875  AbsCmp<double>()));
14876 
14877  oomph_info << "\n Max/min error: " << mmesh_pt->max_error() << " "
14878  << mmesh_pt->min_error() << std::endl;
14879  }
14880  else
14881  {
14882  oomph_info << "Info/Warning: Mesh adaptation is disabled."
14883  << std::endl;
14884  }
14885  }
14886  else
14887  {
14888  oomph_info << "Info/Warning: Mesh cannot be adapted" << std::endl;
14889  }
14890  }
14891 
14892  // Multiple submeshes
14893  //------------------
14894  else
14895  {
14896  // Resize to the number of submeshes
14897  elemental_error.resize(Nmesh);
14898 
14899  // Loop over submeshes
14900  for (unsigned imesh = 0; imesh < Nmesh; imesh++)
14901  {
14902  // Refine single mesh uniformly if possible
14903  if (RefineableMeshBase* mmesh_pt =
14904  dynamic_cast<RefineableMeshBase*>(Problem::mesh_pt(imesh)))
14905  {
14906  // Get pointer to error estimator
14907  ErrorEstimator* error_estimator_pt =
14908  mmesh_pt->spatial_error_estimator_pt();
14909 
14910 #ifdef PARANOID
14911  if (error_estimator_pt == 0)
14912  {
14913  throw OomphLibError("Error estimator hasn't been set yet",
14914  OOMPH_CURRENT_FUNCTION,
14915  OOMPH_EXCEPTION_LOCATION);
14916  }
14917 #endif
14918  // If we can adapt the mesh
14919  if (mmesh_pt->is_adaptation_enabled())
14920  {
14921  // Get error for all elements
14922  elemental_error[imesh].resize(mmesh_pt->nelement());
14923  if (mmesh_pt->doc_info_pt() == 0)
14924  {
14925  error_estimator_pt->get_element_errors(Problem::mesh_pt(imesh),
14926  elemental_error[imesh]);
14927  }
14928  else
14929  {
14930  error_estimator_pt->get_element_errors(Problem::mesh_pt(imesh),
14931  elemental_error[imesh],
14932  *mmesh_pt->doc_info_pt());
14933  }
14934 
14935  // Store max./min error
14936  mmesh_pt->max_error() =
14937  std::fabs(*std::max_element(elemental_error[imesh].begin(),
14938  elemental_error[imesh].end(),
14939  AbsCmp<double>()));
14940 
14941  mmesh_pt->min_error() =
14942  std::fabs(*std::min_element(elemental_error[imesh].begin(),
14943  elemental_error[imesh].end(),
14944  AbsCmp<double>()));
14945 
14946  oomph_info << "\n Max/min error: " << mmesh_pt->max_error() << " "
14947  << mmesh_pt->min_error() << std::endl;
14948  }
14949  else
14950  {
14951  oomph_info << "Info/Warning: Mesh adaptation is disabled."
14952  << std::endl;
14953  }
14954  }
14955  else
14956  {
14957  oomph_info << "Info/Warning: Mesh cannot be adapted." << std::endl;
14958  }
14959 
14960  } // End of loop over submeshes
14961  }
14962  }
14963 
14964  //========================================================================
14965  /// Get max and min error for all elements in submeshes
14966  //========================================================================
14968  {
14969  // Get the bifurcation type
14970  int bifurcation_type = this->Assembly_handler_pt->bifurcation_type();
14971  // If we are tracking a bifurcation then call the bifurcation adapt function
14972  if (bifurcation_type != 0)
14973  {
14974  this->bifurcation_adapt_doc_errors(bifurcation_type);
14975  // Return immediately
14976  return;
14977  }
14978 
14979  // Number of submeshes?
14980  unsigned Nmesh = nsub_mesh();
14981 
14982  // Single mesh:
14983  //------------
14984  if (Nmesh == 0)
14985  {
14986  // Is the single mesh refineable?
14987  if (RefineableMeshBase* mmesh_pt =
14988  dynamic_cast<RefineableMeshBase*>(mesh_pt(0)))
14989  {
14990  // Get pointer to error estimator
14991  ErrorEstimator* error_estimator_pt =
14992  mmesh_pt->spatial_error_estimator_pt();
14993 
14994 #ifdef PARANOID
14995  if (error_estimator_pt == 0)
14996  {
14997  throw OomphLibError("Error estimator hasn't been set yet",
14998  OOMPH_CURRENT_FUNCTION,
14999  OOMPH_EXCEPTION_LOCATION);
15000  }
15001 #endif
15002 
15003  // Get error for all elements
15004  Vector<double> elemental_error(mmesh_pt->nelement());
15005  if (!doc_info.is_doc_enabled())
15006  {
15007  error_estimator_pt->get_element_errors(mesh_pt(0), elemental_error);
15008  }
15009  else
15010  {
15011  error_estimator_pt->get_element_errors(
15012  mesh_pt(0), elemental_error, doc_info);
15013  }
15014 
15015  // Store max./min actual error
15016  mmesh_pt->max_error() = std::fabs(*std::max_element(
15017  elemental_error.begin(), elemental_error.end(), AbsCmp<double>()));
15018 
15019  mmesh_pt->min_error() = std::fabs(*std::min_element(
15020  elemental_error.begin(), elemental_error.end(), AbsCmp<double>()));
15021 
15022  oomph_info << "\n Max/min error: " << mmesh_pt->max_error() << " "
15023  << mmesh_pt->min_error() << std::endl;
15024  }
15025  }
15026 
15027  // Multiple submeshes
15028  //------------------
15029  else
15030  {
15031  // Loop over submeshes
15032  for (unsigned imesh = 0; imesh < Nmesh; imesh++)
15033  {
15034  // Is the single mesh refineable?
15035  if (RefineableMeshBase* mmesh_pt =
15036  dynamic_cast<RefineableMeshBase*>(mesh_pt(imesh)))
15037  {
15038  // Get pointer to error estimator
15039  ErrorEstimator* error_estimator_pt =
15040  mmesh_pt->spatial_error_estimator_pt();
15041 
15042 #ifdef PARANOID
15043  if (error_estimator_pt == 0)
15044  {
15045  throw OomphLibError("Error estimator hasn't been set yet",
15046  OOMPH_CURRENT_FUNCTION,
15047  OOMPH_EXCEPTION_LOCATION);
15048  }
15049 #endif
15050 
15051  // Get error for all elements
15052  Vector<double> elemental_error(mmesh_pt->nelement());
15053  if (mmesh_pt->doc_info_pt() == 0)
15054  {
15055  error_estimator_pt->get_element_errors(mesh_pt(imesh),
15056  elemental_error);
15057  }
15058  else
15059  {
15060  error_estimator_pt->get_element_errors(
15061  mesh_pt(imesh), elemental_error, *mmesh_pt->doc_info_pt());
15062  }
15063 
15064  // Store max./min error if the mesh has any elements
15065  if (mesh_pt(imesh)->nelement() > 0)
15066  {
15067  mmesh_pt->max_error() =
15068  std::fabs(*std::max_element(elemental_error.begin(),
15069  elemental_error.end(),
15070  AbsCmp<double>()));
15071 
15072  mmesh_pt->min_error() =
15073  std::fabs(*std::min_element(elemental_error.begin(),
15074  elemental_error.end(),
15075  AbsCmp<double>()));
15076  }
15077 
15078  oomph_info << "\n Max/min error: " << mmesh_pt->max_error() << " "
15079  << mmesh_pt->min_error() << std::endl;
15080  }
15081 
15082  } // End of loop over submeshes
15083  }
15084  }
15085 
15086  //========================================================================
15087  /// Refine (one and only!) mesh by splitting the elements identified
15088  /// by their numbers relative to the problems' only mesh, then rebuild
15089  /// the problem.
15090  //========================================================================
15092  const Vector<unsigned>& elements_to_be_refined)
15093  {
15095 
15096  // Number of submeshes?
15097  unsigned Nmesh = nsub_mesh();
15098 
15099  // Single mesh:
15100  if (Nmesh == 0)
15101  {
15102  // Refine single mesh if possible
15103  if (TreeBasedRefineableMeshBase* mmesh_pt =
15104  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(0)))
15105  {
15106  mmesh_pt->refine_selected_elements(elements_to_be_refined);
15107  }
15108  else
15109  {
15110  oomph_info << "Info/Warning: Mesh cannot be refined " << std::endl;
15111  }
15112  }
15113  // Multiple submeshes
15114  else
15115  {
15116  std::ostringstream error_message;
15117  error_message << "Problem::refine_selected_elements(...) only works for\n"
15118  << "multiple-mesh problems if you specify the mesh\n"
15119  << "number in the function argument before the Vector,\n"
15120  << "or a Vector of Vectors for each submesh.\n"
15121  << std::endl;
15122  throw OomphLibError(
15123  error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
15124  }
15125 
15126  // Any actions after the adapatation phase
15128 
15129  // Attach the boundary conditions to the mesh
15130  oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
15131  }
15132 
15133  //========================================================================
15134  /// Refine (one and only!) mesh by splitting the elements identified
15135  /// by their pointers, then rebuild the problem.
15136  //========================================================================
15138  const Vector<RefineableElement*>& elements_to_be_refined_pt)
15139  {
15141 
15142  // Number of submeshes?
15143  unsigned Nmesh = nsub_mesh();
15144 
15145  // Single mesh:
15146  if (Nmesh == 0)
15147  {
15148  // Refine single mesh if possible
15149  if (TreeBasedRefineableMeshBase* mmesh_pt =
15150  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(0)))
15151  {
15152  mmesh_pt->refine_selected_elements(elements_to_be_refined_pt);
15153  }
15154  else
15155  {
15156  oomph_info << "Info/Warning: Mesh cannot be refined " << std::endl;
15157  }
15158  }
15159  // Multiple submeshes
15160  else
15161  {
15162  std::ostringstream error_message;
15163  error_message << "Problem::refine_selected_elements(...) only works for\n"
15164  << "multiple-mesh problems if you specify the mesh\n"
15165  << "number in the function argument before the Vector,\n"
15166  << "or a Vector of Vectors for each submesh.\n"
15167  << std::endl;
15168  throw OomphLibError(
15169  error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
15170  }
15171 
15172  // Any actions after the adapatation phase
15174 
15175  // Do equation numbering
15176  oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
15177  }
15178 
15179  //========================================================================
15180  /// Refine specified submesh by splitting the elements identified
15181  /// by their numbers relative to the specified mesh, then rebuild the problem.
15182  //========================================================================
15184  const unsigned& i_mesh, const Vector<unsigned>& elements_to_be_refined)
15185  {
15187 
15188  // Number of submeshes?
15189  unsigned n_mesh = nsub_mesh();
15190 
15191  if (i_mesh >= n_mesh)
15192  {
15193  std::ostringstream error_message;
15194  error_message << "Problem only has " << n_mesh
15195  << " submeshes. Cannot refine submesh " << i_mesh
15196  << std::endl;
15197  throw OomphLibError(
15198  error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
15199  }
15200 
15201  // Refine single mesh if possible
15202  if (TreeBasedRefineableMeshBase* mmesh_pt =
15203  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(i_mesh)))
15204  {
15205  mmesh_pt->refine_selected_elements(elements_to_be_refined);
15206  }
15207  else
15208  {
15209  oomph_info << "Info/Warning: Mesh cannot be refined " << std::endl;
15210  }
15211 
15212  if (n_mesh > 1)
15213  {
15214  // Rebuild the global mesh
15216  }
15217 
15218  // Any actions after the adapatation phase
15220 
15221  // Do equation numbering
15222  oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
15223  }
15224 
15225 
15226  //========================================================================
15227  /// Refine specified submesh by splitting the elements identified
15228  /// by their pointers, then rebuild the problem.
15229  //========================================================================
15231  const unsigned& i_mesh,
15232  const Vector<RefineableElement*>& elements_to_be_refined_pt)
15233  {
15235 
15236  // Number of submeshes?
15237  unsigned n_mesh = nsub_mesh();
15238 
15239  if (i_mesh >= n_mesh)
15240  {
15241  std::ostringstream error_message;
15242  error_message << "Problem only has " << n_mesh
15243  << " submeshes. Cannot refine submesh " << i_mesh
15244  << std::endl;
15245  throw OomphLibError(
15246  error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
15247  }
15248 
15249  // Refine single mesh if possible
15250  if (TreeBasedRefineableMeshBase* mmesh_pt =
15251  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(i_mesh)))
15252  {
15253  mmesh_pt->refine_selected_elements(elements_to_be_refined_pt);
15254  }
15255  else
15256  {
15257  oomph_info << "Info/Warning: Mesh cannot be refined " << std::endl;
15258  }
15259 
15260  if (n_mesh > 1)
15261  {
15262  // Rebuild the global mesh
15264  }
15265 
15266  // Any actions after the adapatation phase
15268 
15269  // Do equation numbering
15270  oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
15271  }
15272 
15273  //========================================================================
15274  /// Refine all submeshes by splitting the elements identified by their
15275  /// numbers relative to each submesh in a Vector of Vectors, then
15276  /// rebuild the problem.
15277  //========================================================================
15279  const Vector<Vector<unsigned>>& elements_to_be_refined)
15280  {
15282 
15283  // Number of submeshes?
15284  unsigned n_mesh = nsub_mesh();
15285 
15286  // Refine all submeshes if possible
15287  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
15288  {
15289  if (TreeBasedRefineableMeshBase* mmesh_pt =
15290  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(i_mesh)))
15291  {
15292  mmesh_pt->refine_selected_elements(elements_to_be_refined[i_mesh]);
15293  }
15294  else
15295  {
15296  oomph_info << "Info/Warning: Mesh cannot be refined " << std::endl;
15297  }
15298  }
15299 
15300  // Rebuild the global mesh
15302 
15303  // Any actions after the adapatation phase
15305 
15306  // Do equation numbering
15307  oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
15308  }
15309 
15310  //========================================================================
15311  /// Refine all submeshes by splitting the elements identified by their
15312  /// pointers within each submesh in a Vector of Vectors, then
15313  /// rebuild the problem.
15314  //========================================================================
15316  const Vector<Vector<RefineableElement*>>& elements_to_be_refined_pt)
15317  {
15319 
15320  // Number of submeshes?
15321  unsigned n_mesh = nsub_mesh();
15322 
15323  // Refine all submeshes if possible
15324  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
15325  {
15326  if (TreeBasedRefineableMeshBase* mmesh_pt =
15327  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(i_mesh)))
15328  {
15329  mmesh_pt->refine_selected_elements(elements_to_be_refined_pt[i_mesh]);
15330  }
15331  else
15332  {
15333  oomph_info << "Info/Warning: Mesh cannot be refined " << std::endl;
15334  }
15335  }
15336 
15337  // Rebuild the global mesh
15339 
15340  // Any actions after the adapatation phase
15342 
15343  // Do equation numbering
15344  oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
15345  }
15346 
15347  //========================================================================
15348  /// p-refine (one and only!) mesh by refining the elements identified
15349  /// by their numbers relative to the problems' only mesh, then rebuild
15350  /// the problem.
15351  //========================================================================
15353  const Vector<unsigned>& elements_to_be_refined)
15354  {
15356 
15357  // Number of submeshes?
15358  unsigned Nmesh = nsub_mesh();
15359 
15360  // Single mesh:
15361  if (Nmesh == 0)
15362  {
15363  // Refine single mesh if possible
15364  if (TreeBasedRefineableMeshBase* mmesh_pt =
15365  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(0)))
15366  {
15367  mmesh_pt->p_refine_selected_elements(elements_to_be_refined);
15368  }
15369  else
15370  {
15371  oomph_info << "Info/Warning: Mesh cannot be refined " << std::endl;
15372  }
15373  }
15374  // Multiple submeshes
15375  else
15376  {
15377  std::ostringstream error_message;
15378  error_message
15379  << "Problem::p_refine_selected_elements(...) only works for\n"
15380  << "multiple-mesh problems if you specify the mesh\n"
15381  << "number in the function argument before the Vector,\n"
15382  << "or a Vector of Vectors for each submesh.\n"
15383  << std::endl;
15384  throw OomphLibError(
15385  error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
15386  }
15387 
15388  // Any actions after the adapatation phase
15390 
15391  // Attach the boundary conditions to the mesh
15392  oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
15393  }
15394 
15395  //========================================================================
15396  /// p-refine (one and only!) mesh by refining the elements identified
15397  /// by their pointers, then rebuild the problem.
15398  //========================================================================
15400  const Vector<PRefineableElement*>& elements_to_be_refined_pt)
15401  {
15403 
15404  // Number of submeshes?
15405  unsigned Nmesh = nsub_mesh();
15406 
15407  // Single mesh:
15408  if (Nmesh == 0)
15409  {
15410  // Refine single mesh if possible
15411  if (TreeBasedRefineableMeshBase* mmesh_pt =
15412  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(0)))
15413  {
15414  mmesh_pt->p_refine_selected_elements(elements_to_be_refined_pt);
15415  }
15416  else
15417  {
15418  oomph_info << "Info/Warning: Mesh cannot be refined " << std::endl;
15419  }
15420  }
15421  // Multiple submeshes
15422  else
15423  {
15424  std::ostringstream error_message;
15425  error_message
15426  << "Problem::p_refine_selected_elements(...) only works for\n"
15427  << "multiple-mesh problems if you specify the mesh\n"
15428  << "number in the function argument before the Vector,\n"
15429  << "or a Vector of Vectors for each submesh.\n"
15430  << std::endl;
15431  throw OomphLibError(
15432  error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
15433  }
15434 
15435  // Any actions after the adapatation phase
15437 
15438  // Do equation numbering
15439  oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
15440  }
15441 
15442  //========================================================================
15443  /// p-refine specified submesh by refining the elements identified
15444  /// by their numbers relative to the specified mesh, then rebuild the problem.
15445  //========================================================================
15447  const unsigned& i_mesh, const Vector<unsigned>& elements_to_be_refined)
15448  {
15450  "p-refinement for multiple submeshes has not yet been tested.",
15451  "Problem::p_refine_selected_elements()",
15452  OOMPH_EXCEPTION_LOCATION);
15453 
15455 
15456  // Number of submeshes?
15457  unsigned n_mesh = nsub_mesh();
15458 
15459  if (i_mesh >= n_mesh)
15460  {
15461  std::ostringstream error_message;
15462  error_message << "Problem only has " << n_mesh
15463  << " submeshes. Cannot p-refine submesh " << i_mesh
15464  << std::endl;
15465  throw OomphLibError(
15466  error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
15467  }
15468 
15469  // Refine single mesh if possible
15470  if (TreeBasedRefineableMeshBase* mmesh_pt =
15471  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(i_mesh)))
15472  {
15473  mmesh_pt->p_refine_selected_elements(elements_to_be_refined);
15474  }
15475  else
15476  {
15477  oomph_info << "Info/Warning: Mesh cannot be refined " << std::endl;
15478  }
15479 
15480  if (n_mesh > 1)
15481  {
15482  // Rebuild the global mesh
15484  }
15485 
15486  // Any actions after the adapatation phase
15488 
15489  // Do equation numbering
15490  oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
15491  }
15492 
15493 
15494  //========================================================================
15495  /// p-refine specified submesh by refining the elements identified
15496  /// by their pointers, then rebuild the problem.
15497  //========================================================================
15499  const unsigned& i_mesh,
15500  const Vector<PRefineableElement*>& elements_to_be_refined_pt)
15501  {
15503  "p-refinement for multiple submeshes has not yet been tested.",
15504  "Problem::p_refine_selected_elements()",
15505  OOMPH_EXCEPTION_LOCATION);
15506 
15508 
15509  // Number of submeshes?
15510  unsigned n_mesh = nsub_mesh();
15511 
15512  if (i_mesh >= n_mesh)
15513  {
15514  std::ostringstream error_message;
15515  error_message << "Problem only has " << n_mesh
15516  << " submeshes. Cannot p-refine submesh " << i_mesh
15517  << std::endl;
15518  throw OomphLibError(
15519  error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
15520  }
15521 
15522  // Refine single mesh if possible
15523  if (TreeBasedRefineableMeshBase* mmesh_pt =
15524  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(i_mesh)))
15525  {
15526  mmesh_pt->p_refine_selected_elements(elements_to_be_refined_pt);
15527  }
15528  else
15529  {
15530  oomph_info << "Info/Warning: Mesh cannot be refined " << std::endl;
15531  }
15532 
15533  if (n_mesh > 1)
15534  {
15535  // Rebuild the global mesh
15537  }
15538 
15539  // Any actions after the adapatation phase
15541 
15542  // Do equation numbering
15543  oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
15544  }
15545 
15546  //========================================================================
15547  /// p-refine all submeshes by refining the elements identified by their
15548  /// numbers relative to each submesh in a Vector of Vectors, then
15549  /// rebuild the problem.
15550  //========================================================================
15552  const Vector<Vector<unsigned>>& elements_to_be_refined)
15553  {
15555  "p-refinement for multiple submeshes has not yet been tested.",
15556  "Problem::p_refine_selected_elements()",
15557  OOMPH_EXCEPTION_LOCATION);
15558 
15560 
15561  // Number of submeshes?
15562  unsigned n_mesh = nsub_mesh();
15563 
15564  // Refine all submeshes if possible
15565  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
15566  {
15567  if (TreeBasedRefineableMeshBase* mmesh_pt =
15568  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(i_mesh)))
15569  {
15570  mmesh_pt->p_refine_selected_elements(elements_to_be_refined[i_mesh]);
15571  }
15572  else
15573  {
15574  oomph_info << "Info/Warning: Mesh cannot be refined " << std::endl;
15575  }
15576  }
15577 
15578  // Rebuild the global mesh
15580 
15581  // Any actions after the adapatation phase
15583 
15584  // Do equation numbering
15585  oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
15586  }
15587 
15588  //========================================================================
15589  /// p-refine all submeshes by refining the elements identified by their
15590  /// pointers within each submesh in a Vector of Vectors, then
15591  /// rebuild the problem.
15592  //========================================================================
15594  const Vector<Vector<PRefineableElement*>>& elements_to_be_refined_pt)
15595  {
15597  "p-refinement for multiple submeshes has not yet been tested.",
15598  "Problem::p_refine_selected_elements()",
15599  OOMPH_EXCEPTION_LOCATION);
15600 
15602 
15603  // Number of submeshes?
15604  unsigned n_mesh = nsub_mesh();
15605 
15606  // Refine all submeshes if possible
15607  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
15608  {
15609  if (TreeBasedRefineableMeshBase* mmesh_pt =
15610  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(i_mesh)))
15611  {
15612  mmesh_pt->p_refine_selected_elements(elements_to_be_refined_pt[i_mesh]);
15613  }
15614  else
15615  {
15616  oomph_info << "Info/Warning: Mesh cannot be refined " << std::endl;
15617  }
15618  }
15619 
15620  // Rebuild the global mesh
15622 
15623  // Any actions after the adapatation phase
15625 
15626  // Do equation numbering
15627  oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
15628  }
15629 
15630 
15631  //========================================================================
15632  /// Helper function to do compund refinement of (all) refineable
15633  /// (sub)mesh(es) uniformly as many times as specified in vector and
15634  /// rebuild problem; doc refinement process. Set boolean argument
15635  /// to true if you want to prune immediately after refining the meshes
15636  /// individually.
15637  //========================================================================
15638  void Problem::refine_uniformly_aux(const Vector<unsigned>& nrefine_for_mesh,
15639  DocInfo& doc_info,
15640  const bool& prune)
15641  {
15642  double t_start = 0.0;
15644  {
15645  t_start = TimingHelpers::timer();
15646  }
15647 
15649 
15650  double t_end = 0.0;
15652  {
15653  t_end = TimingHelpers::timer();
15654  oomph_info
15655  << "Time for actions before adapt in Problem::refine_uniformly_aux(): "
15656  << t_end - t_start << std::endl;
15657  t_start = TimingHelpers::timer();
15658  }
15659 
15660  // Number of submeshes?
15661  unsigned n_mesh = nsub_mesh();
15662 
15663  // Single mesh:
15664  if (n_mesh == 0)
15665  {
15666  // Refine single mesh uniformly if possible
15667  if (RefineableMeshBase* mmesh_pt =
15668  dynamic_cast<RefineableMeshBase*>(mesh_pt(0)))
15669  {
15670  unsigned nref = nrefine_for_mesh[0];
15671  for (unsigned i = 0; i < nref; i++)
15672  {
15673  mmesh_pt->refine_uniformly(doc_info);
15674  }
15675  }
15676  else
15677  {
15678  oomph_info << "Info/Warning: Mesh cannot be refined uniformly "
15679  << std::endl;
15680  }
15681  }
15682  // Multiple submeshes
15683  else
15684  {
15685  // Loop over submeshes
15686  for (unsigned imesh = 0; imesh < n_mesh; imesh++)
15687  {
15688  // Refine i-th submesh uniformly if possible
15689  if (RefineableMeshBase* mmesh_pt =
15690  dynamic_cast<RefineableMeshBase*>(mesh_pt(imesh)))
15691  {
15692  unsigned nref = nrefine_for_mesh[imesh];
15693  for (unsigned i = 0; i < nref; i++)
15694  {
15695  mmesh_pt->refine_uniformly(doc_info);
15696  }
15697  }
15698  else
15699  {
15700  oomph_info << "Info/Warning: Cannot refine mesh " << imesh
15701  << std::endl;
15702  }
15703  }
15704  // Rebuild the global mesh
15706  }
15707 
15709  {
15710  t_end = TimingHelpers::timer();
15711  oomph_info << "Time for mesh-level mesh refinement in "
15712  << "Problem::refine_uniformly_aux(): " << t_end - t_start
15713  << std::endl;
15714  t_start = TimingHelpers::timer();
15715  }
15716 
15717  // Any actions after the adaptation phase
15719 
15720 
15722  {
15723  t_end = TimingHelpers::timer();
15724  oomph_info
15725  << "Time for actions after adapt Problem::refine_uniformly_aux(): "
15726  << t_end - t_start << std::endl;
15727  t_start = TimingHelpers::timer();
15728  }
15729 
15730 
15731 #ifdef OOMPH_HAS_MPI
15732 
15733  // Prune it?
15734  if (prune)
15735  {
15736  // Note: This calls assign eqn numbers already...
15740 
15742  {
15743  t_end = TimingHelpers::timer();
15744  oomph_info << "Time for Problem::prune_halo_elements_and_nodes() in "
15745  << "Problem::refine_uniformly_aux(): " << t_end - t_start
15746  << std::endl;
15747  }
15748  }
15749  else
15750 #else
15751  if (prune)
15752  {
15753  std::ostringstream error_message;
15754  error_message
15755  << "Requested pruning in serial build. Ignoring the request.\n";
15756  OomphLibWarning(error_message.str(),
15757  "Problem::refine_uniformly_aux()",
15758  OOMPH_EXCEPTION_LOCATION);
15759  }
15760 #endif
15761  {
15762  // Do equation numbering
15763  oomph_info
15764  << "Number of equations after Problem::refine_uniformly_aux(): "
15765  << assign_eqn_numbers() << std::endl;
15766 
15768  {
15769  t_end = TimingHelpers::timer();
15770  oomph_info << "Time for Problem::assign_eqn_numbers() in "
15771  << "Problem::refine_uniformly_aux(): " << t_end - t_start
15772  << std::endl;
15773  }
15774  }
15775  }
15776 
15777 
15778  //========================================================================
15779  /// Helper function to do compund p-refinement of (all) p-refineable
15780  /// (sub)mesh(es) uniformly as many times as specified in vector and
15781  /// rebuild problem; doc refinement process. Set boolean argument
15782  /// to true if you want to prune immediately after refining the meshes
15783  /// individually.
15784  //========================================================================
15786  DocInfo& doc_info,
15787  const bool& prune)
15788  {
15789  double t_start = 0.0;
15791  {
15792  t_start = TimingHelpers::timer();
15793  }
15794 
15796 
15797  double t_end = 0.0;
15799  {
15800  t_end = TimingHelpers::timer();
15801  oomph_info << "Time for actions before adapt in "
15802  "Problem::p_refine_uniformly_aux(): "
15803  << t_end - t_start << std::endl;
15804  t_start = TimingHelpers::timer();
15805  }
15806 
15807  // Number of submeshes?
15808  unsigned n_mesh = nsub_mesh();
15809 
15810  // Single mesh:
15811  if (n_mesh == 0)
15812  {
15813  // Refine single mesh uniformly if possible
15814  if (RefineableMeshBase* mmesh_pt =
15815  dynamic_cast<RefineableMeshBase*>(mesh_pt(0)))
15816  {
15817  unsigned nref = nrefine_for_mesh[0];
15818  for (unsigned i = 0; i < nref; i++)
15819  {
15820  mmesh_pt->p_refine_uniformly(doc_info);
15821  }
15822  }
15823  else
15824  {
15825  oomph_info << "Info/Warning: Mesh cannot be p-refined uniformly "
15826  << std::endl;
15827  }
15828  }
15829  // Multiple submeshes
15830  else
15831  {
15833  "p-refinement for multiple submeshes has not yet been tested.",
15834  "Problem::p_refine_uniformly_aux()",
15835  OOMPH_EXCEPTION_LOCATION);
15836 
15837  // Loop over submeshes
15838  for (unsigned imesh = 0; imesh < n_mesh; imesh++)
15839  {
15840  // Refine i-th submesh uniformly if possible
15841  if (RefineableMeshBase* mmesh_pt =
15842  dynamic_cast<RefineableMeshBase*>(mesh_pt(imesh)))
15843  {
15844  unsigned nref = nrefine_for_mesh[imesh];
15845  for (unsigned i = 0; i < nref; i++)
15846  {
15847  mmesh_pt->p_refine_uniformly(doc_info);
15848  }
15849  }
15850  else
15851  {
15852  oomph_info << "Info/Warning: Cannot p-refine mesh " << imesh
15853  << std::endl;
15854  }
15855  }
15856  // Rebuild the global mesh
15858  }
15859 
15861  {
15862  t_end = TimingHelpers::timer();
15863  oomph_info << "Time for mesh-level mesh refinement in "
15864  << "Problem::p_refine_uniformly_aux(): " << t_end - t_start
15865  << std::endl;
15866  t_start = TimingHelpers::timer();
15867  }
15868 
15869  // Any actions after the adaptation phase
15871 
15872 
15874  {
15875  t_end = TimingHelpers::timer();
15876  oomph_info
15877  << "Time for actions after adapt Problem::p_refine_uniformly_aux(): "
15878  << t_end - t_start << std::endl;
15879  t_start = TimingHelpers::timer();
15880  }
15881 
15882 
15883 #ifdef OOMPH_HAS_MPI
15884 
15885  // Prune it?
15886  if (prune)
15887  {
15888  // Note: This calls assign eqn numbers already...
15892 
15894  {
15895  t_end = TimingHelpers::timer();
15896  oomph_info << "Time for Problem::prune_halo_elements_and_nodes() in "
15897  << "Problem::p_refine_uniformly_aux(): " << t_end - t_start
15898  << std::endl;
15899  }
15900  }
15901  else
15902 #else
15903  if (prune)
15904  {
15905  std::ostringstream error_message;
15906  error_message
15907  << "Requested pruning in serial build. Ignoring the request.\n";
15908  OomphLibWarning(error_message.str(),
15909  "Problem::p_refine_uniformly_aux()",
15910  OOMPH_EXCEPTION_LOCATION);
15911  }
15912 #endif
15913  {
15914  // Do equation numbering
15915  oomph_info
15916  << "Number of equations after Problem::p_refine_uniformly_aux(): "
15917  << assign_eqn_numbers() << std::endl;
15918 
15920  {
15921  t_end = TimingHelpers::timer();
15922  oomph_info << "Time for Problem::assign_eqn_numbers() in "
15923  << "Problem::p_refine_uniformly_aux(): " << t_end - t_start
15924  << std::endl;
15925  }
15926  }
15927  }
15928 
15929  //========================================================================
15930  /// Refine submesh i_mesh uniformly and rebuild problem;
15931  /// doc refinement process.
15932  //========================================================================
15933  void Problem::refine_uniformly(const unsigned& i_mesh, DocInfo& doc_info)
15934  {
15936 
15937 #ifdef PARANOID
15938  // Number of submeshes?
15939  if (i_mesh >= nsub_mesh())
15940  {
15941  std::ostringstream error_message;
15942  error_message << "imesh " << i_mesh
15943  << " is greater than the number of sub meshes "
15944  << nsub_mesh() << std::endl;
15945 
15946  throw OomphLibError(
15947  error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
15948  }
15949 #endif
15950 
15951  // Refine single mesh uniformly if possible
15952  if (RefineableMeshBase* mmesh_pt =
15953  dynamic_cast<RefineableMeshBase*>(mesh_pt(i_mesh)))
15954  {
15955  mmesh_pt->refine_uniformly(doc_info);
15956  }
15957  else
15958  {
15959  oomph_info << "Info/Warning: Mesh cannot be refined uniformly "
15960  << std::endl;
15961  }
15962 
15963  // Rebuild the global mesh
15965 
15966  // Any actions after the adaptation phase
15968 
15969  // Do equation numbering
15970  oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
15971  }
15972 
15973  //========================================================================
15974  /// p-refine submesh i_mesh uniformly and rebuild problem;
15975  /// doc refinement process.
15976  //========================================================================
15977  void Problem::p_refine_uniformly(const unsigned& i_mesh, DocInfo& doc_info)
15978  {
15980 
15981 #ifdef PARANOID
15982  // Number of submeshes?
15983  if (i_mesh >= nsub_mesh())
15984  {
15985  std::ostringstream error_message;
15986  error_message << "imesh " << i_mesh
15987  << " is greater than the number of sub meshes "
15988  << nsub_mesh() << std::endl;
15989 
15990  throw OomphLibError(
15991  error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
15992  }
15993 #endif
15994 
15995  // Refine single mesh uniformly if possible
15996  if (RefineableMeshBase* mmesh_pt =
15997  dynamic_cast<RefineableMeshBase*>(mesh_pt(i_mesh)))
15998  {
15999  mmesh_pt->p_refine_uniformly(doc_info);
16000  }
16001  else
16002  {
16003  oomph_info << "Info/Warning: Mesh cannot be refined uniformly "
16004  << std::endl;
16005  }
16006 
16007  // Rebuild the global mesh
16009 
16010  // Any actions after the adaptation phase
16012 
16013  // Do equation numbering
16014  oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
16015  }
16016 
16017 
16018  //========================================================================
16019  /// Unrefine (all) refineable (sub)mesh(es) uniformly and rebuild problem.
16020  /// Return 0 for success,
16021  /// 1 for failure (if unrefinement has reached the coarsest permitted
16022  /// level)
16023  //========================================================================
16025  {
16026  // Call actions_before_adapt()
16028 
16029  // Has unrefinement been successful?
16030  unsigned success_flag = 0;
16031 
16032  // Number of submeshes?
16033  unsigned n_mesh = nsub_mesh();
16034 
16035  // Single mesh:
16036  if (n_mesh == 0)
16037  {
16038  // Unrefine single mesh uniformly if possible
16039  if (RefineableMeshBase* mmesh_pt =
16040  dynamic_cast<RefineableMeshBase*>(mesh_pt(0)))
16041  {
16042  success_flag += mmesh_pt->unrefine_uniformly();
16043  }
16044  else
16045  {
16046  oomph_info << "Info/Warning: Mesh cannot be unrefined uniformly "
16047  << std::endl;
16048  }
16049  }
16050  // Multiple submeshes
16051  else
16052  {
16053  // Loop over submeshes
16054  for (unsigned imesh = 0; imesh < n_mesh; imesh++)
16055  {
16056  // Unrefine i-th submesh uniformly if possible
16057  if (RefineableMeshBase* mmesh_pt =
16058  dynamic_cast<RefineableMeshBase*>(mesh_pt(imesh)))
16059  {
16060  success_flag += mmesh_pt->unrefine_uniformly();
16061  }
16062  else
16063  {
16064  oomph_info << "Info/Warning: Cannot unrefine mesh " << imesh
16065  << std::endl;
16066  }
16067  }
16068  // Rebuild the global mesh
16070  }
16071 
16072  // Any actions after the adaptation phase
16074 
16075  // Do equation numbering
16076  oomph_info << " Number of equations: " << assign_eqn_numbers() << std::endl;
16077 
16078  // Judge success
16079  if (success_flag > 0)
16080  {
16081  return 1;
16082  }
16083  else
16084  {
16085  return 0;
16086  }
16087  }
16088 
16089  //========================================================================
16090  /// Unrefine submesh i_mesh uniformly and rebuild problem.
16091  /// Return 0 for success,
16092  /// 1 for failure (if unrefinement has reached the coarsest permitted
16093  /// level)
16094  //========================================================================
16095  unsigned Problem::unrefine_uniformly(const unsigned& i_mesh)
16096  {
16098 
16099  // Has unrefinement been successful?
16100  unsigned success_flag = 0;
16101 
16102 #ifdef PARANOID
16103  // Number of submeshes?
16104  if (i_mesh >= nsub_mesh())
16105  {
16106  std::ostringstream error_message;
16107  error_message << "imesh " << i_mesh
16108  << " is greater than the number of sub meshes "
16109  << nsub_mesh() << std::endl;
16110 
16111  throw OomphLibError(
16112  error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
16113  }
16114 #endif
16115 
16116  // Unrefine single mesh uniformly if possible
16117  if (RefineableMeshBase* mmesh_pt =
16118  dynamic_cast<RefineableMeshBase*>(mesh_pt(i_mesh)))
16119  {
16120  success_flag += mmesh_pt->unrefine_uniformly();
16121  }
16122  else
16123  {
16124  oomph_info << "Info/Warning: Mesh cannot be unrefined uniformly "
16125  << std::endl;
16126  }
16127 
16128  // Rebuild the global mesh
16130 
16131  // Any actions after the adaptation phase
16133 
16134  // Do equation numbering
16135  oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
16136 
16137  // Judge success
16138  if (success_flag > 0)
16139  {
16140  return 1;
16141  }
16142  else
16143  {
16144  return 0;
16145  }
16146  }
16147 
16148 
16149  //========================================================================
16150  /// p-unrefine (all) p-refineable (sub)mesh(es) uniformly and rebuild problem;
16151  /// doc refinement process.
16152  //========================================================================
16154  {
16156 
16157  // Number of submeshes?
16158  unsigned n_mesh = nsub_mesh();
16159 
16160  // Single mesh:
16161  if (n_mesh == 0)
16162  {
16163  // Unrefine single mesh uniformly if possible
16164  if (RefineableMeshBase* mmesh_pt =
16165  dynamic_cast<RefineableMeshBase*>(mesh_pt(0)))
16166  {
16167  mmesh_pt->p_unrefine_uniformly(doc_info);
16168  }
16169  else
16170  {
16171  oomph_info << "Info/Warning: Mesh cannot be p-unrefined uniformly "
16172  << std::endl;
16173  }
16174  }
16175  // Multiple submeshes
16176  else
16177  {
16178  // Not tested:
16179  throw OomphLibError("This functionality has not yet been tested.",
16180  OOMPH_CURRENT_FUNCTION,
16181  OOMPH_EXCEPTION_LOCATION);
16182  // Loop over submeshes
16183  for (unsigned imesh = 0; imesh < n_mesh; imesh++)
16184  {
16185  // Unrefine i-th submesh uniformly if possible
16186  if (RefineableMeshBase* mmesh_pt =
16187  dynamic_cast<RefineableMeshBase*>(mesh_pt(imesh)))
16188  {
16189  mmesh_pt->p_unrefine_uniformly(doc_info);
16190  }
16191  else
16192  {
16193  oomph_info << "Info/Warning: Cannot p-unrefine mesh " << imesh
16194  << std::endl;
16195  }
16196  }
16197  // Rebuild the global mesh
16199  }
16200 
16201  // Any actions after the adaptation phase
16203 
16204  // Do equation numbering
16205  oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
16206  }
16207 
16208  //========================================================================
16209  /// p-unrefine submesh i_mesh uniformly and rebuild problem;
16210  /// doc refinement process.
16211  //========================================================================
16212  void Problem::p_unrefine_uniformly(const unsigned& i_mesh, DocInfo& doc_info)
16213  {
16215 
16216 #ifdef PARANOID
16217  // Number of submeshes?
16218  if (i_mesh >= nsub_mesh())
16219  {
16220  std::ostringstream error_message;
16221  error_message << "imesh " << i_mesh
16222  << " is greater than the number of sub meshes "
16223  << nsub_mesh() << std::endl;
16224 
16225  throw OomphLibError(
16226  error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
16227  }
16228 #endif
16229 
16230  // Refine single mesh uniformly if possible
16231  if (RefineableMeshBase* mmesh_pt =
16232  dynamic_cast<RefineableMeshBase*>(mesh_pt(i_mesh)))
16233  {
16234  mmesh_pt->p_unrefine_uniformly(doc_info);
16235  }
16236  else
16237  {
16238  oomph_info << "Info/Warning: Mesh cannot be p-unrefined uniformly "
16239  << std::endl;
16240  }
16241 
16242  // Rebuild the global mesh
16244 
16245  // Any actions after the adaptation phase
16247 
16248  // Do equation numbering
16249  oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
16250  }
16251 
16252 
16253  //========================================================================
16254  /// Do one timestep, dt, forward using Newton's method with specified
16255  /// tolerance and linear solver specified via member data.
16256  /// Keep adapting on all meshes to criteria specified in
16257  /// these meshes (up to max_adapt adaptations are performed).
16258  /// If first_timestep==true, re-set initial conditions after mesh adaptation.
16259  /// Shifting of time can be suppressed by overwriting the
16260  /// default value of shift (true). [Shifting must be done
16261  /// if first_timestep==true because we're constantly re-assigning
16262  /// the initial conditions; if first_timestep==true and shift==false
16263  /// shifting is performed anyway and a warning is issued.
16264  //========================================================================
16265  void Problem::unsteady_newton_solve(const double& dt,
16266  const unsigned& max_adapt,
16267  const bool& first_timestep,
16268  const bool& shift)
16269  {
16270  // Do shifting or not?
16271  bool shift_it = shift;
16272 
16273  // Warning:
16274  if (first_timestep && (!shift) && (!Default_set_initial_condition_called))
16275  {
16276  shift_it = true;
16277  oomph_info
16278  << "\n\n===========================================================\n";
16279  oomph_info << " ******** WARNING *********** \n";
16280  oomph_info
16281  << "===========================================================\n";
16282  oomph_info << "Problem::unsteady_newton_solve() called with "
16283  << std::endl;
16284  oomph_info << "first_timestep: " << first_timestep << std::endl;
16285  oomph_info << "shift: " << shift << std::endl;
16286  oomph_info << "This doesn't make sense (shifting does have to be done"
16287  << std::endl;
16288  oomph_info << "since we're constantly re-assigning the initial conditions"
16289  << std::endl;
16290  oomph_info
16291  << "\n===========================================================\n\n";
16292  }
16293 
16294 
16295  // Find the initial time
16296  double initial_time = time_pt()->time();
16297 
16298  // Max number of solves
16299  unsigned max_solve = max_adapt + 1;
16300 
16301  // Adaptation loop
16302  //----------------
16303  for (unsigned isolve = 0; isolve < max_solve; isolve++)
16304  {
16305  // Only adapt after the first solve has been done!
16306  if (isolve > 0)
16307  {
16308  unsigned n_refined;
16309  unsigned n_unrefined;
16310 
16311  // Adapt problem
16312  adapt(n_refined, n_unrefined);
16313 
16314 #ifdef OOMPH_HAS_MPI
16315  // Adaptation only converges if ALL the processes have no
16316  // refinement or unrefinement to perform
16317  unsigned total_refined = 0;
16318  unsigned total_unrefined = 0;
16320  {
16321  MPI_Allreduce(&n_refined,
16322  &total_refined,
16323  1,
16324  MPI_UNSIGNED,
16325  MPI_SUM,
16326  this->communicator_pt()->mpi_comm());
16327  n_refined = total_refined;
16328  MPI_Allreduce(&n_unrefined,
16329  &total_unrefined,
16330  1,
16331  MPI_UNSIGNED,
16332  MPI_SUM,
16333  this->communicator_pt()->mpi_comm());
16334  n_unrefined = total_unrefined;
16335  }
16336 #endif
16337 
16338  oomph_info << "---> " << n_refined << " elements were refined, and "
16339  << n_unrefined << " were unrefined, in total." << std::endl;
16340 
16341  // Check convergence of adaptation cycle
16342  if ((n_refined == 0) && (n_unrefined == 0))
16343  {
16344  oomph_info << "\n \n Solution is fully converged in "
16345  << "Problem::unsteady_newton_solver() \n \n ";
16346  break;
16347  }
16348 
16349  // Reset the time
16350  time_pt()->time() = initial_time;
16351 
16352  // Reset the inital condition on refined meshes. Note that because we
16353  // have reset the global time to the initial time, the initial
16354  // conditions are reset at time t=0 rather than at time t=dt
16355  if (first_timestep)
16356  {
16357  // Reset default set_initial_condition has been called flag to false
16359 
16360  oomph_info << "Re-setting initial condition " << std::endl;
16362 
16363  // If the default set_initial_condition function has been called,
16364  // we must not shift the timevalues on the first timestep, as we
16365  // will NOT be constantly re-assigning the initial condition
16367  {
16368  shift_it = false;
16369  }
16370  }
16371  }
16372 
16373  // Now do the actual unsteady timestep
16374  // If it's the first time around the loop, or the first timestep
16375  // shift the timevalues, otherwise don't
16376  // Note: we need to shift if it's the first timestep because
16377  // we're constantly re-assigning the initial condition above!
16378  // The only exception to this is if the default set_initial_condition
16379  // function has been called, in which case we must NOT shift!
16380  if ((isolve == 0) || (first_timestep))
16381  {
16382  Problem::unsteady_newton_solve(dt, shift_it);
16383  }
16384  // Subsequent solve: Have shifted already -- don't do it again.
16385  else
16386  {
16387  shift_it = false;
16388  Problem::unsteady_newton_solve(dt, shift_it);
16389  }
16390 
16391  if (isolve == max_solve - 1)
16392  {
16393  oomph_info
16394  << std::endl
16395  << "----------------------------------------------------------"
16396  << std::endl
16397  << "Reached max. number of adaptations in \n"
16398  << "Problem::unsteady_newton_solver().\n"
16399  << "----------------------------------------------------------"
16400  << std::endl
16401  << std::endl;
16402  }
16403 
16404  } // End of adaptation loop
16405  }
16406 
16407 
16408  //========================================================================
16409  /// Adaptive Newton solver.
16410  /// The linear solver takes a pointer to the problem (which defines
16411  /// the Jacobian \b J and the residual Vector \b r) and returns
16412  /// the solution \b x of the system
16413  /// \f[ {\bf J} {\bf x} = - \bf{r} \f].
16414  /// Performs at most max_adapt adaptations on all meshes.
16415  //========================================================================
16416  void Problem::newton_solve(const unsigned& max_adapt)
16417  {
16418  // Max number of solves
16419  unsigned max_solve = max_adapt + 1;
16420 
16421  // Adaptation loop
16422  //----------------
16423  for (unsigned isolve = 0; isolve < max_solve; isolve++)
16424  {
16425  // Only adapt after the first solve has been done!
16426  if (isolve > 0)
16427  {
16428  unsigned n_refined;
16429  unsigned n_unrefined;
16430 
16431  // Adapt problem
16432  adapt(n_refined, n_unrefined);
16433 
16434 #ifdef OOMPH_HAS_MPI
16435  // Adaptation only converges if ALL the processes have no
16436  // refinement or unrefinement to perform
16437  unsigned total_refined = 0;
16438  unsigned total_unrefined = 0;
16440  {
16441  MPI_Allreduce(&n_refined,
16442  &total_refined,
16443  1,
16444  MPI_UNSIGNED,
16445  MPI_SUM,
16446  this->communicator_pt()->mpi_comm());
16447  n_refined = total_refined;
16448  MPI_Allreduce(&n_unrefined,
16449  &total_unrefined,
16450  1,
16451  MPI_UNSIGNED,
16452  MPI_SUM,
16453  this->communicator_pt()->mpi_comm());
16454  n_unrefined = total_unrefined;
16455  }
16456 #endif
16457 
16458  oomph_info << "---> " << n_refined << " elements were refined, and "
16459  << n_unrefined << " were unrefined"
16460 #ifdef OOMPH_HAS_MPI
16461  << ", in total (over all processors).\n";
16462 #else
16463  << ".\n";
16464 #endif
16465 
16466 
16467  // Check convergence of adaptation cycle
16468  if ((n_refined == 0) && (n_unrefined == 0))
16469  {
16470  oomph_info << "\n \n Solution is fully converged in "
16471  << "Problem::newton_solver(). \n \n ";
16472  break;
16473  }
16474  }
16475 
16476 
16477  // Do actual solve
16478  //----------------
16479  {
16480  // Now update anything that needs updating
16481  // NOT NEEDED -- IS CALLED IN newton_solve BELOW! #
16482  // actions_before_newton_solve();
16483 
16484  try
16485  {
16486  // Solve the non-linear problem for this timestep with Newton's method
16487  newton_solve();
16488  }
16489  // Catch any exceptions thrown in the Newton solver
16490  catch (NewtonSolverError& error)
16491  {
16492  oomph_info << std::endl
16493  << "USER-DEFINED ERROR IN NEWTON SOLVER " << std::endl;
16494  // Check to see whether we have reached Max_iterations
16495  if (error.iterations == Max_newton_iterations)
16496  {
16497  oomph_info << "MAXIMUM NUMBER OF ITERATIONS (" << error.iterations
16498  << ") REACHED WITHOUT CONVERGENCE " << std::endl;
16499  }
16500  // If not, it must be that we have exceeded the maximum residuals
16501  else
16502  {
16503  oomph_info << "MAXIMUM RESIDUALS: " << error.maxres
16504  << "EXCEEDS PREDEFINED MAXIMUM " << Max_residuals
16505  << std::endl;
16506  }
16507 
16508  // Die horribly!!
16509  std::ostringstream error_stream;
16510  error_stream << "Error occured in adaptive Newton solver. "
16511  << std::endl;
16512  throw OomphLibError(error_stream.str(),
16513  OOMPH_CURRENT_FUNCTION,
16514  OOMPH_EXCEPTION_LOCATION);
16515  }
16516 
16517  // Now update anything that needs updating
16518  // NOT NEEDED -- WAS CALLED IN newton_solve ABOVE
16519  // !actions_after_newton_solve();
16520 
16521  } // End of solve block
16522 
16523 
16524  if (isolve == max_solve - 1)
16525  {
16526  oomph_info
16527  << std::endl
16528  << "----------------------------------------------------------"
16529  << std::endl
16530  << "Reached max. number of adaptations in \n"
16531  << "Problem::newton_solver().\n"
16532  << "----------------------------------------------------------"
16533  << std::endl
16534  << std::endl;
16535  }
16536 
16537  } // End of adaptation loop
16538  }
16539 
16540  //========================================================================
16541  /// Delete any external storage for any submeshes
16542  /// NB this would ordinarily take place within the adaptation procedure
16543  /// for each submesh (See RefineableMesh::adapt_mesh(...)), but there
16544  /// are instances where the actions_before/after_adapt routines are used
16545  /// and no adaptive routines are called in between (e.g. when doc-ing
16546  /// errors at the end of an adaptive newton solver)
16547  //========================================================================
16549  {
16550  // Number of submeshes
16551  unsigned n_mesh = nsub_mesh();
16552 
16553  // External storage will only exist if there is more than one (sub)mesh
16554  if (n_mesh > 1)
16555  {
16556  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
16557  {
16559  }
16560  }
16561  }
16562 
16563 
16564 #ifdef OOMPH_HAS_MPI
16565 
16566  //====================================================================
16567  /// Get all the halo data stored on this processor and store pointers
16568  /// to the data in a map, indexed by the gobal eqn number
16569  //====================================================================
16570  void Problem::get_all_halo_data(std::map<unsigned, double*>& map_of_halo_data)
16571  {
16572  // Halo data is stored in the meshes, so kick the problem down to that
16573  // level
16574 
16575  // Find the number of meshes
16576  unsigned n_mesh = this->nsub_mesh();
16577  // If there are no submeshes it's only the main mesh
16578  if (n_mesh == 0)
16579  {
16580  mesh_pt()->get_all_halo_data(map_of_halo_data);
16581  }
16582  // Otherwise loop over all the submeshes
16583  else
16584  {
16585  for (unsigned imesh = 0; imesh < n_mesh; ++imesh)
16586  {
16587  mesh_pt(imesh)->get_all_halo_data(map_of_halo_data);
16588  }
16589  }
16590  }
16591 
16592 
16593  //========================================================================
16594  /// Check the halo/haloed/shared node/element schemes.
16595  //========================================================================
16597  {
16598  // The bulk of the stuff that was in this routine is mesh-based, and
16599  // should therefore drop into the Mesh base class. All that needs to remain
16600  // here is a "wrapper" which calls the function dependent upon the number
16601  // of (sub)meshes that may have been distributed.
16602 
16603  unsigned n_mesh = nsub_mesh();
16604 
16605  if (n_mesh == 0)
16606  {
16607  oomph_info << "Checking halo schemes on single mesh" << std::endl;
16608  doc_info.label() = "_one_and_only_mesh_";
16609  mesh_pt()->check_halo_schemes(doc_info,
16611  }
16612  else // there are submeshes
16613  {
16614  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
16615  {
16616  oomph_info << "Checking halo schemes on submesh " << i_mesh
16617  << std::endl;
16618  std::stringstream tmp;
16619  tmp << "_mesh" << i_mesh << "_";
16620  doc_info.label() = tmp.str();
16621  mesh_pt(i_mesh)->check_halo_schemes(doc_info,
16623  }
16624  }
16625  }
16626 
16627 
16628  //========================================================================
16629  /// Synchronise all dofs by calling the appropriate synchronisation
16630  /// routines for all meshes and the assembly handler
16631  //========================================================================
16633  {
16634  // Synchronise dofs themselves
16635  bool do_halos = true;
16636  bool do_external_halos = false;
16637  this->synchronise_dofs(do_halos, do_external_halos);
16638 
16639 
16640  do_halos = false;
16641  do_external_halos = true;
16642  this->synchronise_dofs(do_halos, do_external_halos);
16643 
16644  // Now perform any synchronisation required by the assembly handler
16645  this->assembly_handler_pt()->synchronise();
16646  }
16647 
16648 
16649  //========================================================================
16650  /// Synchronise the degrees of freedom by overwriting
16651  /// the haloed values with their non-halo counterparts held
16652  /// on other processors. Bools control if we deal with data associated with
16653  /// external halo/ed elements/nodes or the "normal" halo/ed ones.
16654  //========================================================================
16655  void Problem::synchronise_dofs(const bool& do_halos,
16656  const bool& do_external_halos)
16657  {
16658  // Do we have submeshes?
16659  unsigned n_mesh_loop = 1;
16660  unsigned nmesh = nsub_mesh();
16661  if (nmesh > 0)
16662  {
16663  n_mesh_loop = nmesh;
16664  }
16665 
16666  // Local storage for number of processors and current processor
16667  const int n_proc = this->communicator_pt()->nproc();
16668 
16669  // If only one processor then return
16670  if (n_proc == 1)
16671  {
16672  return;
16673  }
16674 
16675  const int my_rank = this->communicator_pt()->my_rank();
16676 
16677  // Storage for number of data to be sent to each processor
16678  Vector<int> send_n(n_proc, 0);
16679 
16680  // Storage for all values to be sent to all processors
16681  Vector<double> send_data;
16682 
16683  // Start location within send_data for data to be sent to each processor
16684  Vector<int> send_displacement(n_proc, 0);
16685 
16686  // Loop over all processors
16687  for (int rank = 0; rank < n_proc; rank++)
16688  {
16689  // Set the offset for the current processor
16690  send_displacement[rank] = send_data.size();
16691 
16692  // Don't bother to do anything if the processor in the loop is the
16693  // current processor
16694  if (rank != my_rank)
16695  {
16696  // Deal with sub-meshes one-by-one if required
16697  Mesh* my_mesh_pt = 0;
16698 
16699  // Loop over submeshes
16700  for (unsigned imesh = 0; imesh < n_mesh_loop; imesh++)
16701  {
16702  if (nmesh == 0)
16703  {
16704  my_mesh_pt = mesh_pt();
16705  }
16706  else
16707  {
16708  my_mesh_pt = mesh_pt(imesh);
16709  }
16710 
16711  if (do_halos)
16712  {
16713  // How many of my nodes are haloed by the processor whose values
16714  // are updated?
16715  unsigned n_nod = my_mesh_pt->nhaloed_node(rank);
16716  for (unsigned n = 0; n < n_nod; n++)
16717  {
16718  // Add the data for each haloed node to the vector
16719  my_mesh_pt->haloed_node_pt(rank, n)->add_values_to_vector(
16720  send_data);
16721  }
16722 
16723  // Now loop over haloed elements and prepare to add their
16724  // internal data to the big vector to be sent
16725  Vector<GeneralisedElement*> haloed_elem_pt =
16726  my_mesh_pt->haloed_element_pt(rank);
16727  unsigned nelem_haloed = haloed_elem_pt.size();
16728  for (unsigned e = 0; e < nelem_haloed; e++)
16729  {
16730  haloed_elem_pt[e]->add_internal_data_values_to_vector(send_data);
16731  }
16732  }
16733 
16734  if (do_external_halos)
16735  {
16736  // How many of my nodes are externally haloed by the processor whose
16737  // values are updated? NB these nodes are on the external mesh.
16738  unsigned n_ext_nod = my_mesh_pt->nexternal_haloed_node(rank);
16739  for (unsigned n = 0; n < n_ext_nod; n++)
16740  {
16741  // Add data from each external haloed node to the vector
16742  my_mesh_pt->external_haloed_node_pt(rank, n)
16743  ->add_values_to_vector(send_data);
16744  }
16745 
16746  // Now loop over haloed elements and prepare to send internal data
16747  unsigned next_elem_haloed =
16748  my_mesh_pt->nexternal_haloed_element(rank);
16749  for (unsigned e = 0; e < next_elem_haloed; e++)
16750  {
16751  my_mesh_pt->external_haloed_element_pt(rank, e)
16753  }
16754  }
16755  } // end of loop over meshes
16756  }
16757 
16758  // Find the number of data added to the vector
16759  send_n[rank] = send_data.size() - send_displacement[rank];
16760  }
16761 
16762 
16763  // Storage for the number of data to be received from each processor
16764  Vector<int> receive_n(n_proc, 0);
16765 
16766  // Now send numbers of data to be sent between all processors
16767  MPI_Alltoall(&send_n[0],
16768  1,
16769  MPI_INT,
16770  &receive_n[0],
16771  1,
16772  MPI_INT,
16773  this->communicator_pt()->mpi_comm());
16774 
16775  // We now prepare the data to be received
16776  // by working out the displacements from the received data
16777  Vector<int> receive_displacement(n_proc, 0);
16778  int receive_data_count = 0;
16779  for (int rank = 0; rank < n_proc; ++rank)
16780  {
16781  // Displacement is number of data received so far
16782  receive_displacement[rank] = receive_data_count;
16783  receive_data_count += receive_n[rank];
16784  }
16785 
16786  // Now resize the receive buffer for all data from all processors
16787  // Make sure that it has a size of at least one
16788  if (receive_data_count == 0)
16789  {
16790  ++receive_data_count;
16791  }
16792  Vector<double> receive_data(receive_data_count);
16793 
16794  // Make sure that the send buffer has size at least one
16795  // so that we don't get a segmentation fault
16796  if (send_data.size() == 0)
16797  {
16798  send_data.resize(1);
16799  }
16800 
16801  // Now send the data between all the processors
16802  MPI_Alltoallv(&send_data[0],
16803  &send_n[0],
16804  &send_displacement[0],
16805  MPI_DOUBLE,
16806  &receive_data[0],
16807  &receive_n[0],
16808  &receive_displacement[0],
16809  MPI_DOUBLE,
16810  this->communicator_pt()->mpi_comm());
16811 
16812  // Now use the received data to update the halo nodes
16813  for (int send_rank = 0; send_rank < n_proc; send_rank++)
16814  {
16815  // Don't bother to do anything for the processor corresponding to the
16816  // current processor or if no data were received from this processor
16817  if ((send_rank != my_rank) && (receive_n[send_rank] != 0))
16818  {
16819  // Counter for the data within the large array
16820  unsigned count = receive_displacement[send_rank];
16821 
16822  // Deal with sub-meshes one-by-one if required
16823  Mesh* my_mesh_pt = 0;
16824 
16825  // Loop over submeshes
16826  for (unsigned imesh = 0; imesh < n_mesh_loop; imesh++)
16827  {
16828  if (nmesh == 0)
16829  {
16830  my_mesh_pt = mesh_pt();
16831  }
16832  else
16833  {
16834  my_mesh_pt = mesh_pt(imesh);
16835  }
16836 
16837  if (do_halos)
16838  {
16839  // How many of my nodes are halos whose non-halo counter
16840  // parts live on processor send_rank?
16841  unsigned n_nod = my_mesh_pt->nhalo_node(send_rank);
16842  for (unsigned n = 0; n < n_nod; n++)
16843  {
16844  // Read in values for each halo node
16845  my_mesh_pt->halo_node_pt(send_rank, n)
16846  ->read_values_from_vector(receive_data, count);
16847  }
16848 
16849  // Get number of halo elements whose non-halo is
16850  // on process send_rank
16851  Vector<GeneralisedElement*> halo_elem_pt =
16852  my_mesh_pt->halo_element_pt(send_rank);
16853 
16854  unsigned nelem_halo = halo_elem_pt.size();
16855  for (unsigned e = 0; e < nelem_halo; e++)
16856  {
16857  halo_elem_pt[e]->read_internal_data_values_from_vector(
16858  receive_data, count);
16859  }
16860  }
16861 
16862  if (do_external_halos)
16863  {
16864  // How many of my nodes are external halos whose external non-halo
16865  // counterparts live on processor send_rank?
16866  unsigned n_ext_nod = my_mesh_pt->nexternal_halo_node(send_rank);
16867 
16868  // Copy into the values of the external halo nodes
16869  // on the present processors
16870  for (unsigned n = 0; n < n_ext_nod; n++)
16871  {
16872  // Read the data from the array into each halo node
16873  my_mesh_pt->external_halo_node_pt(send_rank, n)
16874  ->read_values_from_vector(receive_data, count);
16875  }
16876 
16877  // Get number of halo elements whose non-halo is
16878  // on process send_rank
16879  unsigned next_elem_halo =
16880  my_mesh_pt->nexternal_halo_element(send_rank);
16881  for (unsigned e = 0; e < next_elem_halo; e++)
16882  {
16883  my_mesh_pt->external_halo_element_pt(send_rank, e)
16884  ->read_internal_data_values_from_vector(receive_data, count);
16885  }
16886  }
16887 
16888  } // end of loop over meshes
16889  }
16890  } // End of data is received
16891  } // End of synchronise
16892 
16893 
16894  //========================================================================
16895  /// Synchronise equation numbers and return the total
16896  /// number of degrees of freedom in the overall problem
16897  //========================================================================
16898  long Problem::synchronise_eqn_numbers(const bool& assign_local_eqn_numbers)
16899  {
16900  // number of equations on this processor, which at this stage is only known
16901  // by counting the number of dofs that have been added to the problem
16902  unsigned my_n_eqn = Dof_pt.size();
16903 
16904  // my rank
16905  unsigned my_rank = Communicator_pt->my_rank();
16906 
16907  // number of processors
16908  unsigned nproc = Communicator_pt->nproc();
16909 
16910  // // Time alternative communication
16911  // Vector<unsigned> n_eqn(nproc);
16912  // {
16913  // double t_start = TimingHelpers::timer();
16914 
16915  // // Gather numbers of equations (enumerated independently on all procs)
16916  // MPI_Allgather(&my_n_eqn,1,MPI_UNSIGNED,&n_eqn[0],
16917  // 1,MPI_INT,Communicator_pt->mpi_comm());
16918 
16919  // double t_end = TimingHelpers::timer();
16920  // oomph_info << "Time for allgather-based exchange of eqn numbers: "
16921  // << t_end-t_start << std::endl;
16922  // }
16923 
16924  double t_start = TimingHelpers::timer();
16925 
16926  // send my_n_eqn to with rank greater than my_rank
16927  unsigned n_send = nproc - my_rank - 1;
16928  Vector<MPI_Request> send_req(n_send);
16929  for (unsigned p = my_rank + 1; p < nproc; p++)
16930  {
16931  MPI_Isend(&my_n_eqn,
16932  1,
16933  MPI_UNSIGNED,
16934  p,
16935  0,
16936  Communicator_pt->mpi_comm(),
16937  &send_req[p - my_rank - 1]);
16938  }
16939 
16940  // recv n_eqn from processors with rank less than my_rank
16941  Vector<unsigned> n_eqn_on_proc(my_rank);
16942  for (unsigned p = 0; p < my_rank; p++)
16943  {
16944  MPI_Recv(&n_eqn_on_proc[p],
16945  1,
16946  MPI_UNSIGNED,
16947  p,
16948  0,
16949  Communicator_pt->mpi_comm(),
16950  MPI_STATUS_IGNORE);
16951  }
16952 
16953  double t_end = 0.0;
16955  {
16956  t_end = TimingHelpers::timer();
16957  oomph_info << "Time for send and receive stuff: " << t_end - t_start
16958  << std::endl;
16959  t_start = TimingHelpers::timer();
16960  }
16961 
16962  // determine the number of equation on processors with rank
16963  // less than my_rank
16964  unsigned my_eqn_num_base = 0;
16965  for (unsigned p = 0; p < my_rank; p++)
16966  {
16967  my_eqn_num_base += n_eqn_on_proc[p];
16968  // if (n_eqn_on_proc[p]!=n_eqn[p])
16969  // {
16970  // std::cout << "proc " << my_rank << "clash in eqn numbers: "
16971  // << p << " " << n_eqn_on_proc[p] << " " << n_eqn[p]
16972  // << std::endl;
16973  // }
16974  }
16975 
16976  // Loop over all internal data (on elements) and bump up their
16977  // equation numbers if they exist
16978  unsigned nelem = mesh_pt()->nelement();
16979  for (unsigned e = 0; e < nelem; e++)
16980  {
16981  GeneralisedElement* el_pt = mesh_pt()->element_pt(e);
16982 
16983  unsigned nintern_data = el_pt->ninternal_data();
16984  for (unsigned iintern = 0; iintern < nintern_data; iintern++)
16985  {
16986  Data* int_data_pt = el_pt->internal_data_pt(iintern);
16987  unsigned nval = int_data_pt->nvalue();
16988  for (unsigned ival = 0; ival < nval; ival++)
16989  {
16990  int old_eqn_number = int_data_pt->eqn_number(ival);
16991  if (old_eqn_number >= 0) // i.e. it's being used
16992  {
16993  // Bump up eqn number
16994  int new_eqn_number = old_eqn_number + my_eqn_num_base;
16995  int_data_pt->eqn_number(ival) = new_eqn_number;
16996  }
16997  }
16998  }
16999  }
17000 
17001  // Loop over all nodes on current processor and bump up their
17002  // equation numbers if they're not pinned!
17003  unsigned nnod = mesh_pt()->nnode();
17004  for (unsigned j = 0; j < nnod; j++)
17005  {
17006  Node* nod_pt = mesh_pt()->node_pt(j);
17007 
17008  // loop over ALL eqn numbers - variable number of values
17009  unsigned nval = nod_pt->nvalue();
17010 
17011  for (unsigned ival = 0; ival < nval; ival++)
17012  {
17013  int old_eqn_number = nod_pt->eqn_number(ival);
17014  // Include all eqn numbers
17015  if (old_eqn_number >= 0)
17016  {
17017  // Bump up eqn number
17018  int new_eqn_number = old_eqn_number + my_eqn_num_base;
17019  nod_pt->eqn_number(ival) = new_eqn_number;
17020  }
17021  }
17022 
17023  // Is this a solid node? If so, need to bump up its equation number(s)
17024  SolidNode* solid_nod_pt = dynamic_cast<SolidNode*>(nod_pt);
17025 
17026  if (solid_nod_pt != 0)
17027  {
17028  // Find equation numbers
17029  unsigned nval = solid_nod_pt->variable_position_pt()->nvalue();
17030  for (unsigned ival = 0; ival < nval; ival++)
17031  {
17032  int old_eqn_number =
17033  solid_nod_pt->variable_position_pt()->eqn_number(ival);
17034  // include all eqn numbers
17035 
17036  if (old_eqn_number >= 0)
17037  {
17038  // Bump up eqn number
17039  int new_eqn_number = old_eqn_number + my_eqn_num_base;
17040  solid_nod_pt->variable_position_pt()->eqn_number(ival) =
17041  new_eqn_number;
17042  }
17043  }
17044  }
17045  }
17046 
17048  {
17049  t_end = TimingHelpers::timer();
17050  oomph_info << "Time for bumping: " << t_end - t_start << std::endl;
17051  t_start = TimingHelpers::timer();
17052  }
17053 
17054 
17055  // Now copy the haloed eqn numbers across
17056  // This has to include the internal data equation numbers as well
17057  // as the solid node equation numbers
17058  bool do_halos = true;
17059  bool do_external_halos = false;
17060  copy_haloed_eqn_numbers_helper(do_halos, do_external_halos);
17061 
17063  {
17064  t_end = TimingHelpers::timer();
17065  oomph_info << "Time for copy_haloed_eqn_numbers_helper for halos: "
17066  << t_end - t_start << std::endl;
17067  t_start = TimingHelpers::timer();
17068  }
17069 
17070  // Now do external halo stuff
17071  do_halos = false;
17072  do_external_halos = true;
17073  copy_haloed_eqn_numbers_helper(do_halos, do_external_halos);
17074 
17076  {
17077  t_end = TimingHelpers::timer();
17078  oomph_info
17079  << "Time for copy_haloed_eqn_numbers_helper for external halos: "
17080  << t_end - t_start << std::endl;
17081  t_start = TimingHelpers::timer();
17082  }
17083 
17084  // Now the global equation numbers have been updated.
17085  //---------------------------------------------------
17086  // Setup the local equation numbers again.
17087  //----------------------------------------
17088  if (assign_local_eqn_numbers)
17089  {
17090  // Loop over the submeshes: Note we need to call the submeshes' own
17091  // assign_*_eqn_number() otherwise we miss additional functionality
17092  // that is implemented (e.g.) in SolidMeshes!
17093  unsigned n_sub_mesh = nsub_mesh();
17094  if (n_sub_mesh == 0)
17095  {
17097  }
17098  else
17099  {
17100  for (unsigned i = 0; i < n_sub_mesh; i++)
17101  {
17103  }
17104  }
17105  }
17106 
17108  {
17109  t_end = TimingHelpers::timer();
17110  oomph_info << "Time for assign_local_eqn_numbers in sync: "
17111  << t_end - t_start << std::endl;
17112  t_start = TimingHelpers::timer();
17113  }
17114 
17115  // wait for the sends to complete
17116  if (n_send > 0)
17117  {
17118  Vector<MPI_Status> send_status(n_send);
17119  MPI_Waitall(n_send, &send_req[0], &send_status[0]);
17120  }
17121 
17123  {
17124  t_end = TimingHelpers::timer();
17125  oomph_info << "Time for waitall: " << t_end - t_start << std::endl;
17126  t_start = TimingHelpers::timer();
17127  }
17128 
17129  // build the Dof distribution pt
17130  Dof_distribution_pt->build(Communicator_pt, my_eqn_num_base, my_n_eqn);
17131 
17132  // and return the total number of equations in the problem
17133  return (long)Dof_distribution_pt->nrow();
17134  }
17135 
17136 
17137  //=======================================================================
17138  /// A private helper function to
17139  /// copy the haloed equation numbers into the halo equation numbers,
17140  /// either for the problem's one and only mesh or for all of its
17141  /// submeshes. Bools control if we deal with data associated with
17142  /// external halo/ed elements/nodes or the "normal" halo/ed ones.
17143  //===================================================================
17144  void Problem::copy_haloed_eqn_numbers_helper(const bool& do_halos,
17145  const bool& do_external_halos)
17146  {
17147  // Do we have submeshes?
17148  unsigned n_mesh_loop = 1;
17149  unsigned nmesh = nsub_mesh();
17150  if (nmesh > 0)
17151  {
17152  n_mesh_loop = nmesh;
17153  }
17154 
17155  // Storage for number of processors and current processor
17156  int n_proc = this->communicator_pt()->nproc();
17157 
17158  // If only one processor then return
17159  if (n_proc == 1)
17160  {
17161  return;
17162  }
17163  int my_rank = this->communicator_pt()->my_rank();
17164 
17165  // Storage for number of data to be sent to each processor
17166  Vector<int> send_n(n_proc, 0);
17167  // Storage for all equation numbers to be sent to all processors
17168  Vector<long> send_data;
17169  // Start location within send_data for data to be sent to each processor
17170  Vector<int> send_displacement(n_proc, 0);
17171 
17172 
17173  // Loop over all processors whose eqn numbers are to be updated
17174  for (int rank = 0; rank < n_proc; rank++)
17175  {
17176  // Set the displacement of the current processor in the loop
17177  send_displacement[rank] = send_data.size();
17178 
17179  // If I'm not the processor whose halo eqn numbers are updated,
17180  // some of my nodes may be haloed: Stick their
17181  // eqn numbers into the vector
17182  if (rank != my_rank)
17183  {
17184  // Deal with sub-meshes one-by-one if required
17185  Mesh* my_mesh_pt = 0;
17186 
17187  // Loop over submeshes
17188  for (unsigned imesh = 0; imesh < n_mesh_loop; imesh++)
17189  {
17190  if (nmesh == 0)
17191  {
17192  my_mesh_pt = mesh_pt();
17193  }
17194  else
17195  {
17196  my_mesh_pt = mesh_pt(imesh);
17197  }
17198 
17199  if (do_halos)
17200  {
17201  // Add equation numbers for each haloed node
17202  unsigned n_nod = my_mesh_pt->nhaloed_node(rank);
17203  for (unsigned n = 0; n < n_nod; n++)
17204  {
17205  my_mesh_pt->haloed_node_pt(rank, n)->add_eqn_numbers_to_vector(
17206  send_data);
17207  }
17208 
17209  // Add the equation numbers associated with internal data
17210  // in the haloed elements
17211  Vector<GeneralisedElement*> haloed_elem_pt =
17212  my_mesh_pt->haloed_element_pt(rank);
17213  unsigned nelem_haloed = haloed_elem_pt.size();
17214  for (unsigned e = 0; e < nelem_haloed; e++)
17215  {
17216  haloed_elem_pt[e]->add_internal_eqn_numbers_to_vector(send_data);
17217  }
17218  }
17219 
17220  if (do_external_halos)
17221  {
17222  // Add equation numbers associated with external haloed nodes
17223  unsigned n_ext_nod = my_mesh_pt->nexternal_haloed_node(rank);
17224  for (unsigned n = 0; n < n_ext_nod; n++)
17225  {
17226  my_mesh_pt->external_haloed_node_pt(rank, n)
17227  ->add_eqn_numbers_to_vector(send_data);
17228  }
17229 
17230  // Add the equation numbers associated with internal data in
17231  // each external haloed element
17232  unsigned next_elem_haloed =
17233  my_mesh_pt->nexternal_haloed_element(rank);
17234  for (unsigned e = 0; e < next_elem_haloed; e++)
17235  {
17236  // how many internal data values for this element?
17237  my_mesh_pt->external_haloed_element_pt(rank, e)
17239  }
17240  }
17241 
17242  } // end of loop over meshes
17243  }
17244 
17245  // Find the number of data added to the vector by this processor
17246  send_n[rank] = send_data.size() - send_displacement[rank];
17247  }
17248 
17249  // Storage for the number of data to be received from each processor
17250  Vector<int> receive_n(n_proc, 0);
17251 
17252  // Communicate all numbers of data to be sent between all processors
17253  MPI_Alltoall(&send_n[0],
17254  1,
17255  MPI_INT,
17256  &receive_n[0],
17257  1,
17258  MPI_INT,
17259  this->communicator_pt()->mpi_comm());
17260 
17261  // We now prepare the data to be received
17262  // by working out the displacements from the received data
17263  Vector<int> receive_displacement(n_proc, 0);
17264  int receive_data_count = 0;
17265  for (int rank = 0; rank < n_proc; ++rank)
17266  {
17267  // Displacement is number of data received so far
17268  receive_displacement[rank] = receive_data_count;
17269  receive_data_count += receive_n[rank];
17270  }
17271 
17272  // Now resize the receive buffer
17273  // Make sure that it has a size of at least one
17274  if (receive_data_count == 0)
17275  {
17276  ++receive_data_count;
17277  }
17278  Vector<long> receive_data(receive_data_count);
17279 
17280  // Make sure that the send buffer has size at least one
17281  // so that we don't get a segmentation fault
17282  if (send_data.size() == 0)
17283  {
17284  send_data.resize(1);
17285  }
17286 
17287  // Now send the data between all the processors
17288  MPI_Alltoallv(&send_data[0],
17289  &send_n[0],
17290  &send_displacement[0],
17291  MPI_LONG,
17292  &receive_data[0],
17293  &receive_n[0],
17294  &receive_displacement[0],
17295  MPI_LONG,
17296  this->communicator_pt()->mpi_comm());
17297 
17298 
17299  // Loop over all other processors to receive their
17300  // eqn numbers
17301  for (int send_rank = 0; send_rank < n_proc; send_rank++)
17302  {
17303  // Don't do anything for the processor corresponding to the
17304  // current processor or if no data were received from this processor
17305  if ((send_rank != my_rank) && (receive_n[send_rank] != 0))
17306  {
17307  // Counter for the data within the large array
17308  unsigned count = receive_displacement[send_rank];
17309 
17310  // Deal with sub-meshes one-by-one if required
17311  Mesh* my_mesh_pt = 0;
17312 
17313  // Loop over submeshes
17314  for (unsigned imesh = 0; imesh < n_mesh_loop; imesh++)
17315  {
17316  if (nmesh == 0)
17317  {
17318  my_mesh_pt = mesh_pt();
17319  }
17320  else
17321  {
17322  my_mesh_pt = mesh_pt(imesh);
17323  }
17324 
17325  if (do_halos)
17326  {
17327  // How many of my nodes are halos whose non-halo counter
17328  // parts live on processor send_rank?
17329  unsigned n_nod = my_mesh_pt->nhalo_node(send_rank);
17330  for (unsigned n = 0; n < n_nod; n++)
17331  {
17332  // Generalise to variable number of values per node
17333  my_mesh_pt->halo_node_pt(send_rank, n)
17334  ->read_eqn_numbers_from_vector(receive_data, count);
17335  }
17336 
17337  // Get number of halo elements whose non-halo is on
17338  // process send_rank
17339  Vector<GeneralisedElement*> halo_elem_pt =
17340  my_mesh_pt->halo_element_pt(send_rank);
17341  unsigned nelem_halo = halo_elem_pt.size();
17342  for (unsigned e = 0; e < nelem_halo; e++)
17343  {
17344  halo_elem_pt[e]->read_internal_eqn_numbers_from_vector(
17345  receive_data, count);
17346  }
17347  }
17348 
17349  if (do_external_halos)
17350  {
17351  // How many of my nodes are external halos whose external non-halo
17352  // counterparts live on processor send_rank?
17353  unsigned n_ext_nod = my_mesh_pt->nexternal_halo_node(send_rank);
17354  for (unsigned n = 0; n < n_ext_nod; n++)
17355  {
17356  my_mesh_pt->external_halo_node_pt(send_rank, n)
17357  ->read_eqn_numbers_from_vector(receive_data, count);
17358  }
17359 
17360  // Get number of external halo elements whose external haloed
17361  // counterpart is on process send_rank
17362  unsigned next_elem_halo =
17363  my_mesh_pt->nexternal_halo_element(send_rank);
17364  for (unsigned e = 0; e < next_elem_halo; e++)
17365  {
17366  my_mesh_pt->external_halo_element_pt(send_rank, e)
17367  ->read_internal_eqn_numbers_from_vector(receive_data, count);
17368  }
17369  }
17370 
17371  } // end of loop over meshes
17372  }
17373  } // End of loop over processors
17374  }
17375 
17376  //==========================================================================
17377  /// Balance the load of a (possibly non-uniformly refined) problem that has
17378  /// already been distributed, by re-distributing elements over the processors.
17379  /// Produce explicit stats of load balancing process if boolean, report_stats,
17380  /// is set to true and doc various bits of data (mainly for debugging)
17381  /// in directory specified by DocInfo object.
17382  //==========================================================================
17384  DocInfo& doc_info,
17385  const bool& report_stats,
17386  const Vector<unsigned>& input_target_domain_for_local_non_halo_element)
17387  {
17388  double start_t = TimingHelpers::timer();
17389 
17390  // Number of processes
17391  const unsigned n_proc = this->communicator_pt()->nproc();
17392 
17393  // Don't do anything if this is a single-process job
17394  if (n_proc == 1)
17395  {
17396  if (report_stats)
17397  {
17398  std::ostringstream warn_message;
17399  warn_message << "WARNING: You've tried to load balance a problem over\n"
17400  << "only one processor: ignoring your request.\n";
17401  OomphLibWarning(warn_message.str(),
17402  "Problem::load_balance()",
17403  OOMPH_EXCEPTION_LOCATION);
17404  }
17405  }
17406  // Multiple processors
17407  else
17408  {
17409  // This will only work if the problem has already been distributed
17411  {
17412  // Throw an error
17413  std::ostringstream error_stream;
17414  error_stream << "You have called Problem::load_balance()\n"
17415  << "on a non-distributed problem. This doesn't\n"
17416  << "make sense -- go distribute your problem first."
17417  << std::endl;
17418  throw OomphLibError(
17419  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
17420  }
17421 
17422  // Timings
17423  double t_start = 0.0;
17424  double t_metis = 0.0;
17425  double t_partition = 0.0;
17426  double t_distribute = 0.0;
17427  double t_refine = 0.0;
17428  double t_copy_solution = 0.0;
17429 
17430  if (report_stats)
17431  {
17432  t_start = TimingHelpers::timer();
17433  }
17434 
17435 
17436 #ifdef PARANOID
17437  unsigned old_ndof = ndof();
17438 #endif
17439 
17440  // Store pointers to the old mesh(es) so we retain a handle
17441  //---------------------------------------------------------
17442  // to them for deletion
17443  //---------------------
17444  Vector<Mesh*> old_mesh_pt;
17445  unsigned n_mesh = nsub_mesh();
17446  if (n_mesh == 0)
17447  {
17448  // Resize the container
17449  old_mesh_pt.resize(1);
17450  old_mesh_pt[0] = mesh_pt();
17451  }
17452  else
17453  {
17454  // Resize the container
17455  old_mesh_pt.resize(n_mesh);
17456  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
17457  {
17458  old_mesh_pt[i_mesh] = mesh_pt(i_mesh);
17459  }
17460  }
17461 
17462 
17463  // Partition the global mesh in its current state
17464  //-----------------------------------------------
17465 
17466  // target_domain_for_local_non_halo_element[e] contains the number
17467  // of the domain [0,1,...,nproc-1] to which non-halo element e on THE
17468  // CURRENT PROCESSOR ONLY has been assigned. The order of the non-halo
17469  // elements is the same as in the Problem's mesh, with the halo
17470  // elements being skipped.
17471  Vector<unsigned> target_domain_for_local_non_halo_element;
17472 
17473  // Do any of the processors want to go through externally imposed
17474  // partitioning? If so, we'd better do it here too (even if the processor
17475  // is empty, e.g. following a restart on a larger number of procs) or
17476  // we hang.
17477  unsigned local_ntarget =
17478  input_target_domain_for_local_non_halo_element.size();
17479  unsigned global_ntarget = 0;
17480  MPI_Allreduce(&local_ntarget,
17481  &global_ntarget,
17482  1,
17483  MPI_UNSIGNED,
17484  MPI_MAX,
17485  Communicator_pt->mpi_comm());
17486 
17487  // External prescribed partitioning
17488  if (global_ntarget > 0)
17489  {
17490  target_domain_for_local_non_halo_element =
17491  input_target_domain_for_local_non_halo_element;
17492  }
17493  else
17494  {
17495  // Metis does not always produce repeatable results which is
17496  // a disaster for validation runs -- this bypasses metis and
17497  // comes up with a stupid but repeatable partioning.
17499  {
17500  // Bypass METIS to perform the partitioning
17501  unsigned objective = 0;
17502  bool bypass_metis = true;
17504  this,
17505  objective,
17506  target_domain_for_local_non_halo_element,
17507  bypass_metis);
17508  }
17509  else
17510  {
17511  // Use METIS to perform the partitioning
17512  unsigned objective = 0;
17514  this, objective, target_domain_for_local_non_halo_element);
17515  }
17516  }
17517 
17518  if (report_stats)
17519  {
17520  t_metis = TimingHelpers::timer();
17521  }
17522 
17523  // Setup map linking element with target domain
17524  std::map<GeneralisedElement*, unsigned>
17525  target_domain_for_local_non_halo_element_map;
17526  unsigned n_elem = mesh_pt()->nelement();
17527  unsigned count_non_halo_el = 0;
17528  for (unsigned e = 0; e < n_elem; e++)
17529  {
17530  GeneralisedElement* el_pt = mesh_pt()->element_pt(e);
17531  if (!el_pt->is_halo())
17532  {
17533  target_domain_for_local_non_halo_element_map[el_pt] =
17534  target_domain_for_local_non_halo_element[count_non_halo_el];
17535  count_non_halo_el++;
17536  }
17537  }
17538 
17539  // Load balancing is equivalent to distribution so call the
17540  // appropriate "actions before". NOTE: This acts on the
17541  // current, refined, distributed, etc. problem object
17542  // before it's being wiped. This step is therefore not
17543  // a duplicate of the call below, which acts on the
17544  // new, not-yet refined, distributed etc. problem!
17546 
17547  // Re-setup target domains for remaining elements (FaceElements
17548  // are likely to have been stripped out in actions_before_distribute()
17549  n_elem = mesh_pt()->nelement();
17550  target_domain_for_local_non_halo_element.clear();
17551  target_domain_for_local_non_halo_element.reserve(n_elem);
17552  count_non_halo_el = 0;
17553  for (unsigned e = 0; e < n_elem; e++)
17554  {
17555  GeneralisedElement* el_pt = mesh_pt()->element_pt(e);
17556  if (!el_pt->is_halo())
17557  {
17558  target_domain_for_local_non_halo_element.push_back(
17559  target_domain_for_local_non_halo_element_map[el_pt]);
17560  }
17561  } // for (e < n_elem)
17562 
17563  // Re-setup the number of sub-meshes since some of them may have
17564  // been stripped out in actions_before_distribute(), but save the
17565  // number of old sub-meshes
17566  const unsigned n_old_sub_meshes = n_mesh;
17567  n_mesh = nsub_mesh();
17568 
17569  // Now get the target domains for each of the submeshes, we only
17570  // get the target domains for the nonhalo elements
17571  Vector<Vector<unsigned>> target_domain_for_local_non_halo_element_submesh(
17572  n_mesh);
17573  // If we have no sub-meshes then we do not need to copy the target areas
17574  // of the submeshes
17575  if (n_mesh != 0)
17576  {
17577  // Counter to copy the target domains from the global vector
17578  unsigned count_td = 0;
17579  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
17580  {
17581  // Get the number of elements (considering halo elements)
17582  const unsigned nsub_ele = mesh_pt(i_mesh)->nelement();
17583  // Now copy that number of data from the global target domains
17584  for (unsigned i = 0; i < nsub_ele; i++)
17585  {
17586  // Get the element
17587  GeneralisedElement* ele_pt = mesh_pt(i_mesh)->element_pt(i);
17588  // ... and check if it is a nonhalo element
17589  if (!ele_pt->is_halo())
17590  {
17591  // Get the target domain for the current element
17592  const unsigned target_domain =
17593  target_domain_for_local_non_halo_element[count_td++];
17594  // Add the target domain for the nonhalo element in the
17595  // submesh
17596  target_domain_for_local_non_halo_element_submesh[i_mesh]
17597  .push_back(target_domain);
17598  } // if (!ele_pt->is_halo())
17599  } // for (i < nsub_ele)
17600  } // for (imesh < n_mesh)
17601 
17602 #ifdef PARANOID
17603  // Check that the total number of copied data be the same as the
17604  // total number of nonhalo elements in the (sub)-mesh(es)
17605  const unsigned ntarget_domain =
17606  target_domain_for_local_non_halo_element.size();
17607  if (count_td != ntarget_domain)
17608  {
17609  std::ostringstream error_stream;
17610  error_stream
17611  << "The number of nonhalo elements (" << count_td
17612  << ") found in (all)\n"
17613  << "the (sub)-mesh(es) is different from the number of target "
17614  "domains\n"
17615  << "(" << ntarget_domain << ") for the nonhalo elements.\n"
17616  << "Please ensure that you called the rebuild_global_mesh() method "
17617  << "after the\npossible deletion of FaceElements in "
17618  << "actions_before_distribute()!!!\n\n";
17619  throw OomphLibError(error_stream.str(),
17620  "Problem::load_balance()",
17621  OOMPH_EXCEPTION_LOCATION);
17622  } // if (count_td != ntarget_domain)
17623 #endif
17624 
17625  } // if (n_mesh != 0)
17626 
17627  // Check if we have different type of submeshes (unstructured
17628  // and/or structured). Identify to which type each submesh belongs.
17629  // If we have only one mesh then identify to which type that mesh
17630  // belongs.
17631 
17632  // The load balancing strategy acts in the structured meshes and
17633  // then acts in the unstructured meshes
17634 
17635  // Vector to temporaly store pointers to unstructured meshes
17636  // (TriangleMeshBase)
17637  Vector<TriangleMeshBase*> unstructured_mesh_pt;
17638  std::vector<bool> is_unstructured_mesh;
17639 
17640  // Flag to indicate that there are unstructured meshes as part of
17641  // the problem
17642  bool are_there_unstructured_meshes = false;
17643 
17644  // We have only one mesh
17645  if (n_mesh == 0)
17646  {
17647  // Check if it is a TriangleMeshBase mesh
17648  if (TriangleMeshBase* tri_mesh_pt =
17649  dynamic_cast<TriangleMeshBase*>(old_mesh_pt[0]))
17650  {
17651  // Add the pointer to the unstructured meshes container
17652  unstructured_mesh_pt.push_back(tri_mesh_pt);
17653  // Indicate that it is an unstructured mesh
17654  is_unstructured_mesh.push_back(true);
17655  // Indicate that there are unstructured meshes as part of the
17656  // problem
17657  are_there_unstructured_meshes = true;
17658  }
17659  else
17660  {
17661  // Add the pointer to the unstructured meshes container (null
17662  // pointer)
17663  unstructured_mesh_pt.push_back(tri_mesh_pt);
17664  // Indicate that it is not an unstructured mesh
17665  is_unstructured_mesh.push_back(false);
17666  }
17667  } // if (n_mesh == 0)
17668  else // We have sub-meshes
17669  {
17670  // Check which sub-meshes are unstructured meshes (work with the
17671  // old sub-meshes number)
17672  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
17673  {
17674  // Is it a TriangleMeshBase mesh
17675  if (TriangleMeshBase* tri_mesh_pt =
17676  dynamic_cast<TriangleMeshBase*>(old_mesh_pt[i_mesh]))
17677  {
17678  // Add the pointer to the unstructured meshes container
17679  unstructured_mesh_pt.push_back(tri_mesh_pt);
17680  // Indicate that it is an unstructured mesh
17681  is_unstructured_mesh.push_back(true);
17682  // Indicate that there are unstructured meshes as part of the
17683  // problem
17684  are_there_unstructured_meshes = true;
17685  }
17686  else
17687  {
17688  // Add the pointer to the unstructured meshes container (null
17689  // pointer)
17690  unstructured_mesh_pt.push_back(tri_mesh_pt);
17691  // Indicate that it is not an unstructured mesh
17692  is_unstructured_mesh.push_back(false);
17693  }
17694  } // for (i_mesh < n_mesh)
17695  } // else if (n_mesh == 0) // We have sub-meshes
17696 
17697  // Extract data to be sent to various processors after the
17698  //--------------------------------------------------------
17699  // problem has been rebuilt/re-distributed
17700  //----------------------------------------
17701 
17702  // Storage for number of data to be sent to each processor
17703  Vector<int> send_n(n_proc, 0);
17704 
17705  // Storage for all values to be sent to all processors
17706  Vector<double> send_data;
17707 
17708  // Start location within send_data for data to be sent to each processor
17709  Vector<int> send_displacement(n_proc, 0);
17710 
17711  // Old and new domains for each base element (available for all, for
17712  // convenience)
17713  Vector<unsigned> old_domain_for_base_element;
17714  Vector<unsigned> new_domain_for_base_element;
17715 
17716  // Flat-packed refinement info, labeled by id of locally
17717  // available root elements
17718  std::map<unsigned, Vector<unsigned>> flat_packed_refinement_info_for_root;
17719 
17720  // Max. level of refinement
17721  unsigned max_refinement_level_overall = 0;
17722 
17723  // Prepare the input for the get_data...() method, only copy the
17724  // data from the structured meshes, TreeBaseMesh meshes
17726  target_domain_for_local_non_halo_element_in_structured_mesh;
17727  if (n_mesh == 0)
17728  {
17729  // Check if the mesh is an structured mesh
17730  if (!is_unstructured_mesh[0])
17731  {
17732  const unsigned nele_mesh =
17733  target_domain_for_local_non_halo_element.size();
17734  for (unsigned e = 0; e < nele_mesh; e++)
17735  {
17736  const unsigned target_domain =
17737  target_domain_for_local_non_halo_element[e];
17738  target_domain_for_local_non_halo_element_in_structured_mesh
17739  .push_back(target_domain);
17740  } // for (e < nele_mesh)
17741  } // if (!is_unstructured_mesh[0])
17742  } // if (n_mesh == 0)
17743  else
17744  {
17745  // Copy the target domains from the structured meshes only
17746  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
17747  {
17748  // Check if the mesh is an structured mesh
17749  if (!is_unstructured_mesh[i_mesh])
17750  {
17751  const unsigned nele_sub_mesh =
17752  target_domain_for_local_non_halo_element_submesh[i_mesh].size();
17753  for (unsigned e = 0; e < nele_sub_mesh; e++)
17754  {
17755  const unsigned target_domain =
17756  target_domain_for_local_non_halo_element_submesh[i_mesh][e];
17757  target_domain_for_local_non_halo_element_in_structured_mesh
17758  .push_back(target_domain);
17759  } // for (e < nele_sub_mesh)
17760  } // if (!is_triangle_mesh_base[i_mesh])
17761  } // for (i_mesh < n_mesh)
17762  } // else if (n_mesh == 0)
17763 
17764  // Extract data from current problem
17765  // sorted into data to be sent to various processors after
17766  // rebuilding the meshes in a load-balanced form
17768  target_domain_for_local_non_halo_element_in_structured_mesh,
17769  send_n,
17770  send_data,
17771  send_displacement,
17772  old_domain_for_base_element,
17773  new_domain_for_base_element,
17774  max_refinement_level_overall);
17775 
17776  // Extract flat-packed refinement pattern
17778  old_domain_for_base_element,
17779  new_domain_for_base_element,
17780  max_refinement_level_overall,
17781  flat_packed_refinement_info_for_root);
17782 
17783  if (report_stats)
17784  {
17785  t_partition = TimingHelpers::timer();
17786  oomph_info << "CPU for partition calculation for roots: "
17787  << t_partition - t_metis << std::endl;
17788  }
17789 
17790 
17791  // Flush and delete old submeshes and null the global mesh
17792  //--------------------------------------------------------
17793  // and rebuild the new (not yet distributed, refined etc.) mesh
17794  //-------------------------------------------------------------
17795  // that will be distributed in the new, improved way determined
17796  //-------------------------------------------------------------
17797  // by METIS
17798  //---------
17799  Vector<unsigned> pruned_refinement_level(
17800  std::max(int(n_old_sub_meshes), 1));
17801  if (n_mesh == 0)
17802  {
17803  pruned_refinement_level[0] = 0;
17804  TreeBasedRefineableMeshBase* ref_mesh_pt =
17805  dynamic_cast<TreeBasedRefineableMeshBase*>(old_mesh_pt[0]);
17806  if (ref_mesh_pt != 0)
17807  {
17808  pruned_refinement_level[0] =
17809  ref_mesh_pt->uniform_refinement_level_when_pruned();
17810  }
17811 
17812  // If the mesh is an unstructured mesh (TriangleMeshBase mesh)
17813  // then we should not delete it since the load balance strategy
17814  // requires the mesh
17815 
17816  // Delete the mesh if it is not an unstructured mesh
17817  if (!is_unstructured_mesh[0])
17818  {
17819  delete old_mesh_pt[0];
17820  old_mesh_pt[0] = 0;
17821  } // if (!is_unstructured_mesh[0])
17822  } // if (n_mesh==0)
17823  else
17824  {
17825  // Loop over the number of old meshes (required to delete the
17826  // pointers of structured meshes in the old_mesh_pt structure)
17827  for (unsigned i_mesh = 0; i_mesh < n_old_sub_meshes; i_mesh++)
17828  {
17829  pruned_refinement_level[i_mesh] = 0;
17830  TreeBasedRefineableMeshBase* ref_mesh_pt =
17831  dynamic_cast<TreeBasedRefineableMeshBase*>(old_mesh_pt[i_mesh]);
17832  if (ref_mesh_pt != 0)
17833  {
17834  pruned_refinement_level[i_mesh] =
17835  ref_mesh_pt->uniform_refinement_level_when_pruned();
17836  }
17837 
17838  // If the mesh is an unstructured mesh (TriangleMeshBase mesh)
17839  // then we should NOT delete it since the load balance strategy
17840  // requires the mesh
17841 
17842  // Delete the mesh if it is not an unstructured mesh
17843  if (!is_unstructured_mesh[i_mesh])
17844  {
17845  delete old_mesh_pt[i_mesh];
17846  old_mesh_pt[i_mesh] = 0;
17847  } // if (!is_unstructured_mesh[i_mesh])
17848 
17849  } // for (i_mesh<n_mesh)
17850 
17851  // Empty storage for sub-meshes
17852  flush_sub_meshes();
17853 
17854  // Flush the storage for nodes and elements in compound mesh
17855  // (they've already been deleted in the sub-meshes)
17857 
17858  // Kill
17859  delete mesh_pt();
17860  mesh_pt() = 0;
17861  } // else if (n_mesh==0)
17862 
17863  bool some_mesh_has_been_pruned = false;
17864  unsigned n = pruned_refinement_level.size();
17865  for (unsigned i = 0; i < n; i++)
17866  {
17867  if (pruned_refinement_level[i] > 0) some_mesh_has_been_pruned = true;
17868  }
17869 
17870  // (Re-)build the new mesh(es) -- this must get the problem into the
17871  // state it was in when it was first distributed!
17872  build_mesh();
17873 
17874  // Has one of the meshes been pruned; if so refine to the
17875  // common refinement level
17876  if (some_mesh_has_been_pruned)
17877  {
17878  // Do actions before adapt
17880 
17881  // Re-assign number of submeshes -- when this was first
17882  // set, the problem may have had face meshes that have now
17883  // disappeared.
17884  n_mesh = nsub_mesh();
17885 
17886  // Now adapt meshes manually
17887  if (n_mesh == 0)
17888  {
17889  TreeBasedRefineableMeshBase* ref_mesh_pt =
17890  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt());
17891  if (ref_mesh_pt != 0)
17892  {
17893  // Get min and max refinement level
17894  unsigned local_min_ref = 0;
17895  unsigned local_max_ref = 0;
17896  ref_mesh_pt->get_refinement_levels(local_min_ref, local_max_ref);
17897 
17898  // Reconcile between processors: If (e.g. following
17899  // distribution/pruning) the mesh has no elements on this
17900  // processor) then ignore its contribution to the poll of
17901  // max/min refinement levels
17902  int int_local_min_ref = local_min_ref;
17903  if (ref_mesh_pt->nelement() == 0)
17904  {
17905  int_local_min_ref = INT_MAX;
17906  }
17907  int int_min_ref = 0;
17908  MPI_Allreduce(&int_local_min_ref,
17909  &int_min_ref,
17910  1,
17911  MPI_INT,
17912  MPI_MIN,
17913  Communicator_pt->mpi_comm());
17914 
17915  // Overall min refinement level over all meshes
17916  unsigned min_ref = unsigned(int_min_ref);
17917 
17918  // Refine as many times as required to get refinement up to
17919  // uniform refinement level after last prune
17920  unsigned nref = pruned_refinement_level[0] - min_ref;
17921  oomph_info << "Refining one-and-only mesh uniformly " << nref
17922  << " times\n";
17923  for (unsigned i = 0; i < nref; i++)
17924  {
17925  ref_mesh_pt->refine_uniformly();
17926  }
17927  }
17928  }
17929  else
17930  {
17931  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
17932  {
17933  TreeBasedRefineableMeshBase* ref_mesh_pt =
17934  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(i_mesh));
17935  if (ref_mesh_pt != 0)
17936  {
17937  // Get min and max refinement level
17938  unsigned local_min_ref = 0;
17939  unsigned local_max_ref = 0;
17940  ref_mesh_pt->get_refinement_levels(local_min_ref, local_max_ref);
17941 
17942  // Reconcile between processors: If (e.g. following
17943  // distribution/pruning) the mesh has no elements on this
17944  // processor) then ignore its contribution to the poll of
17945  // max/min refinement levels
17946  int int_local_min_ref = local_min_ref;
17947  if (ref_mesh_pt->nelement() == 0)
17948  {
17949  int_local_min_ref = INT_MAX;
17950  }
17951  int int_min_ref = 0;
17952  MPI_Allreduce(&int_local_min_ref,
17953  &int_min_ref,
17954  1,
17955  MPI_INT,
17956  MPI_MIN,
17957  Communicator_pt->mpi_comm());
17958 
17959  // Overall min refinement level over all meshes
17960  unsigned min_ref = unsigned(int_min_ref);
17961 
17962  // Refine as many times as required to get refinement up to
17963  // uniform refinement level after last prune
17964  unsigned nref = pruned_refinement_level[i_mesh] - min_ref;
17965  oomph_info << "Refining sub-mesh " << i_mesh << " uniformly "
17966  << nref << " times\n";
17967  for (unsigned i = 0; i < nref; i++)
17968  {
17969  ref_mesh_pt->refine_uniformly();
17970  }
17971  }
17972  }
17973  // Rebuild the global mesh
17975  }
17976 
17977  // Do actions after adapt
17979 
17980  // Re-assign number of submeshes -- when this was first
17981  // set, the problem may have had face meshes that have now
17982  // disappeared.
17983  n_mesh = nsub_mesh();
17984  } // if (some_mesh_has_been_pruned)
17985 
17986 
17987  // Perform any actions before distribution but now for the new mesh
17988  // NOTE: This does NOT replicate the actions_before_distribute()
17989  // call made above for the previous mesh!
17991 
17992  // Do some book-keeping
17993  //---------------------
17994 
17995  // Re-assign number of submeshes -- when this was first
17996  // set, the problem may have had face meshes that have now
17997  // disappeared.
17998  n_mesh = nsub_mesh();
17999 
18000  // The submeshes, if they exist, need to know their own element
18001  // domains.
18002  // NOTE: This vector only stores the target domains or the
18003  // element partition for structured meshes
18004  Vector<Vector<unsigned>> submesh_element_partition(n_mesh);
18005  if (n_mesh != 0)
18006  {
18007  unsigned count = 0;
18008  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
18009  {
18010  // Only work with structured meshes
18011  if (!is_unstructured_mesh[i_mesh])
18012  {
18013  // Get the number of element in the mesh
18014  const unsigned nsub_ele = mesh_pt(i_mesh)->nelement();
18015  submesh_element_partition[i_mesh].resize(nsub_ele);
18016  for (unsigned e = 0; e < nsub_ele; e++)
18017  {
18018  submesh_element_partition[i_mesh][e] =
18019  new_domain_for_base_element[count++];
18020  } // for (e<nsub_elem)
18021  } // if (sub_mesh_pt!=0)
18022  } // for (i_mesh<n_mesh)
18023 
18024 #ifdef PARANOID
18025  const unsigned nnew_domain_for_base_element =
18026  new_domain_for_base_element.size();
18027  if (count != nnew_domain_for_base_element)
18028  {
18029  std::ostringstream error_stream;
18030  error_stream
18031  << "The number of READ target domains for nonhalo elements\n"
18032  << " is (" << count << "), but the number of target domains for\n"
18033  << "nonhalo elements is (" << nnew_domain_for_base_element
18034  << ")!\n";
18035  throw OomphLibError(error_stream.str(),
18036  "Problem::load_balance()",
18037  OOMPH_EXCEPTION_LOCATION);
18038  }
18039 #endif
18040 
18041  } // if (n_mesh!=0)
18042 
18043  // Setup the map between "root" element and number in global mesh
18044  // again
18045 
18046  // This map is only established for structured meshes, then we
18047  // need to check here the type of mesh
18048  if (n_mesh == 0)
18049  {
18050  // Check if the only one mesh is an stuctured mesh
18051  if (!is_unstructured_mesh[0])
18052  {
18053  const unsigned n_ele = mesh_pt()->nelement();
18054  Base_mesh_element_pt.resize(n_ele);
18056  for (unsigned e = 0; e < n_ele; e++)
18057  {
18058  GeneralisedElement* el_pt = mesh_pt()->element_pt(e);
18059  Base_mesh_element_number_plus_one[el_pt] = e + 1;
18060  Base_mesh_element_pt[e] = el_pt;
18061  } // for (e<n_ele)
18062  } // if (!is_triangle_mesh_base[0])
18063  } // if (n_mesh==0)
18064  else
18065  {
18066  // If we have submeshes then we only add those elements that
18067  // belong to structured meshes, but first compute the number of
18068  // total elements in the structured sub-meshes
18069  unsigned nglobal_element = 0;
18070  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
18071  {
18072  // Check if mesh is an structured mesh
18073  if (!is_unstructured_mesh[i_mesh])
18074  {
18075  nglobal_element += mesh_pt(i_mesh)->nelement();
18076  } // if (!is_triangle_mesh_base[i_mesh])
18077  } // for (i_mesh<n_mesh)
18078 
18079  // Once computed the number of elements, then resize the
18080  // structure
18081  Base_mesh_element_pt.resize(nglobal_element);
18083  unsigned counter_base = 0;
18084  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
18085  {
18086  // Check if mesh is a structured mesh
18087  if (!is_unstructured_mesh[i_mesh])
18088  {
18089  const unsigned n_ele = mesh_pt(i_mesh)->nelement();
18090  for (unsigned e = 0; e < n_ele; e++)
18091  {
18092  GeneralisedElement* el_pt = mesh_pt(i_mesh)->element_pt(e);
18093  Base_mesh_element_number_plus_one[el_pt] = counter_base + 1;
18094  Base_mesh_element_pt[counter_base] = el_pt;
18095  // Inrease the global element number
18096  counter_base++;
18097  } // for (e<n_ele)
18098  } // if (!is_triangle_mesh_base[i_mesh])
18099  } // for (i_mesh<n_mesh)
18100 
18101 #ifdef PARANOID
18102  if (counter_base != nglobal_element)
18103  {
18104  std::ostringstream error_stream;
18105  error_stream << "The number of global elements (" << nglobal_element
18106  << ") is not the same as the number of\nadded elements ("
18107  << counter_base << ") to the Base_mesh_element_pt data "
18108  << "structure!!!\n\n";
18109  throw OomphLibError(error_stream.str(),
18110  "Problem::load_balance()",
18111  OOMPH_EXCEPTION_LOCATION);
18112  } // if (counter_base != nglobal_element)
18113 #endif // #ifdef PARANOID
18114 
18115  } // else if (n_mesh==0)
18116 
18117  // Storage for the number of face elements in the base mesh --
18118  // element is identified by number of bulk element and face index
18119  // so we can reconstruct it if and when the FaceElements have been wiped
18120  // in actions_before_distribute().
18121  // NOTE: Not really clear (any more) why this is required. Typically
18122  // FaceElements get wiped in actions_before_distribute() so
18123  // at this point there shouldn't be any of them left.
18124  // This is certainly the case in all our currently existing
18125  // test codes. However, I'm too scared to take this out
18126  // in case it does matter (we're not insisting that FaceElements
18127  // are always removed in actions_before_distribute()...).
18128  std::map<unsigned, std::map<int, unsigned>> face_element_number;
18129  unsigned n_element = mesh_pt()->nelement();
18130  for (unsigned e = 0; e < n_element; e++)
18131  {
18132  FaceElement* face_el_pt =
18133  dynamic_cast<FaceElement*>(mesh_pt()->finite_element_pt(e));
18134  if (face_el_pt != 0)
18135  {
18136 #ifdef PARANOID
18137  std::stringstream info;
18138  info << "================================================\n";
18139  info << "INFO: I've come across a FaceElement while \n";
18140  info << " load-balancing a problem. \n";
18141  info << "================================================\n";
18142  oomph_info << info.str() << std::endl;
18143 #endif
18144  FiniteElement* bulk_elem_pt = face_el_pt->bulk_element_pt();
18145  unsigned e_bulk = Base_mesh_element_number_plus_one[bulk_elem_pt];
18146 #ifdef PARANOID
18147  if (e_bulk == 0)
18148  {
18149  throw OomphLibError("Base_mesh_element_number_plus_one[...]=0",
18150  OOMPH_CURRENT_FUNCTION,
18151  OOMPH_EXCEPTION_LOCATION);
18152  }
18153 #endif
18154  e_bulk -= 1;
18155  int face_index = face_el_pt->face_index();
18156  face_element_number[e_bulk][face_index] = e;
18157  }
18158  }
18159 
18160  // Distribute the (sub)meshes
18161  //---------------------------
18162  Vector<GeneralisedElement*> deleted_element_pt;
18163  if (n_mesh == 0)
18164  {
18165  // Only distribute (load balance strategy) if this is an
18166  // structured mesh
18167  if (!is_unstructured_mesh[0])
18168  {
18169 #ifdef PARANOID
18170  if (mesh_pt()->nelement() != new_domain_for_base_element.size())
18171  {
18172  std::ostringstream error_stream;
18173  error_stream << "Distributing one-and-only mesh containing "
18174  << mesh_pt()->nelement() << " elements with info for "
18175  << new_domain_for_base_element.size() << std::endl;
18176  throw OomphLibError(error_stream.str(),
18177  OOMPH_CURRENT_FUNCTION,
18178  OOMPH_EXCEPTION_LOCATION);
18179  }
18180 #endif
18181 
18182  if (report_stats)
18183  {
18184  oomph_info << "Distributing one and only mesh\n"
18185  << "------------------------------" << std::endl;
18186  }
18187 
18188  // No pre-set distribution from restart that may leave some
18189  // processors empty so no need to overrule deletion of elements
18190  bool overrule_keep_as_halo_element_status = false;
18191 
18192  mesh_pt()->distribute(this->communicator_pt(),
18193  new_domain_for_base_element,
18194  deleted_element_pt,
18195  doc_info,
18196  report_stats,
18197  overrule_keep_as_halo_element_status);
18198 
18199  } // if (!is_unstructured_mesh[0])
18200 
18201  } // if (n_mesh==0)
18202  else // There are submeshes, "distribute" each one separately
18203  {
18204  // Rebuild the mesh only if one of the meshes was modified
18205  bool need_to_rebuild_mesh = false;
18206  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
18207  {
18208  // Perform the load balancing based on distribution in the
18209  // structured meshes only
18210  if (!is_unstructured_mesh[i_mesh])
18211  {
18212  if (report_stats)
18213  {
18214  oomph_info << "Distributing submesh " << i_mesh << " of "
18215  << n_mesh << " in total\n"
18216  << "---------------------------------------------"
18217  << std::endl;
18218  }
18219 
18220  // Set the doc_info number to reflect the submesh
18221  doc_info.number() = i_mesh;
18222 
18223  // No pre-set distribution from restart that may leave some
18224  // processors empty so no need to overrule deletion of elements
18225  bool overrule_keep_as_halo_element_status = false;
18226  mesh_pt(i_mesh)->distribute(this->communicator_pt(),
18227  submesh_element_partition[i_mesh],
18228  deleted_element_pt,
18229  doc_info,
18230  report_stats,
18231  overrule_keep_as_halo_element_status);
18232 
18233  // Set the flag to rebuild the global mesh
18234  need_to_rebuild_mesh = true;
18235 
18236  } // if (!is_unstructured_mesh[i_mesh])
18237 
18238  } // for (i_mesh<n_mesh)
18239 
18240  if (need_to_rebuild_mesh)
18241  {
18242  // Rebuild the global mesh
18244  } // if (need_to_rebuild_mesh)
18245 
18246  } // else if (n_mesh==0)
18247 
18248  // Null out information associated with deleted elements
18249  unsigned n_del = deleted_element_pt.size();
18250  for (unsigned e = 0; e < n_del; e++)
18251  {
18252  GeneralisedElement* el_pt = deleted_element_pt[e];
18253  unsigned old_el_number = Base_mesh_element_number_plus_one[el_pt] - 1;
18255  Base_mesh_element_pt[old_el_number] = 0;
18256  }
18257 
18258  // Has one of the meshes been pruned before distribution? If so
18259  // then prune here now
18260  if (some_mesh_has_been_pruned)
18261  {
18262  Vector<GeneralisedElement*> deleted_element_pt;
18263  if (n_mesh == 0)
18264  {
18265  TreeBasedRefineableMeshBase* ref_mesh_pt =
18266  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt());
18267  if (ref_mesh_pt != 0)
18268  {
18269  ref_mesh_pt->prune_halo_elements_and_nodes(
18270  deleted_element_pt, doc_info, report_stats);
18271  }
18272  }
18273  else
18274  {
18275  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
18276  {
18277  TreeBasedRefineableMeshBase* ref_mesh_pt =
18278  dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(i_mesh));
18279  if (ref_mesh_pt != 0)
18280  {
18281  ref_mesh_pt->prune_halo_elements_and_nodes(
18282  deleted_element_pt, doc_info, report_stats);
18283  }
18284  }
18285  // Rebuild the global mesh
18287  }
18288 
18289  // Null out information associated with deleted elements
18290  unsigned n_del = deleted_element_pt.size();
18291  for (unsigned e = 0; e < n_del; e++)
18292  {
18293  GeneralisedElement* el_pt = deleted_element_pt[e];
18294  unsigned old_el_number = Base_mesh_element_number_plus_one[el_pt] - 1;
18296  Base_mesh_element_pt[old_el_number] = 0;
18297  }
18298 
18299  // Setup the map between "root" element and number in global mesh again
18301  }
18302 
18303  if (report_stats)
18304  {
18305  t_distribute = TimingHelpers::timer();
18306  oomph_info << "CPU for build and distribution of new mesh(es): "
18307  << t_distribute - t_partition << std::endl;
18308  }
18309 
18310 
18311  // Send refinement info to other processors
18312  //-----------------------------------------
18313 
18314  // Storage for refinement pattern: Given ID of root element,
18315  // root_element_id, and current refinement level, level, the e-th entry in
18316  // refinement_info_for_root_elements[root_element_id][level][e] is equal
18317 
18318  // to 2 if the e-th element (using the enumeration when the mesh has been
18319  // refined to the level-th level) is to be refined during the next
18320  // refinement; it's 1 if it's not to be refined.
18321  Vector<Vector<Vector<unsigned>>> refinement_info_for_root_elements;
18322 
18323 
18324  // Send refinement information between processors, using flat-packed
18325  // information accumulated earlier
18326  send_refinement_info_helper(old_domain_for_base_element,
18327  new_domain_for_base_element,
18328  max_refinement_level_overall,
18329  flat_packed_refinement_info_for_root,
18330  refinement_info_for_root_elements);
18331 
18332  // Refine each mesh based upon refinement information stored for each root
18333  //------------------------------------------------------------------------
18334  refine_distributed_base_mesh(refinement_info_for_root_elements,
18335  max_refinement_level_overall);
18336 
18337  if (report_stats)
18338  {
18339  t_refine = TimingHelpers::timer();
18340  oomph_info << "CPU for refinement of base mesh: "
18341  << t_refine - t_distribute << std::endl;
18342  }
18343 
18344  // NOTE: The following two calls are important e.g. when
18345  // FaceElements that resize nodes are attached/detached
18346  // after/before adaptation. If we don't attach them
18347  // on the newly built/refined mesh, there isn't enough
18348  // storage for the nodal values that are sent around
18349  // (in a flat-packed format) resulting in total disaster.
18350  // So we attach them first, but then immediatly strip
18351  // them out again because the FaceElements themselves
18352  // will have been stripped out before distribution/adaptation.
18353 
18354  // Do actions after adapt because we have just adapted the mesh.
18356 
18357  // Now strip it back out to get problem into the same state
18358  // it was in when data to be sent was recorded.
18360 
18361  // Send the stored values in each root from the old mesh into the new mesh
18362  //------------------------------------------------------------------------
18364  send_n, send_data, send_displacement);
18365 
18366  // If there are unstructured meshes here we perform the load
18367  // balancing of those meshes
18368  if (are_there_unstructured_meshes)
18369  {
18370  // Delete any storage of external elements and nodes
18372 
18373  if (n_mesh == 0)
18374  {
18375  // Before doing the load balancing delete the mesh created at
18376  // calling build_mesh(), and restore the pointer to the old
18377  // mesh
18378 
18379  // It MUST be an unstructured mesh, otherwise we should not be
18380  // here
18381  if (is_unstructured_mesh[0])
18382  {
18383  // Delete the new created mesh
18384  delete mesh_pt();
18385  // Re-assign the pointer to the old mesh
18386  this->mesh_pt() = old_mesh_pt[0];
18387  } // if (is_unstructured_mesh[0])
18388 #ifdef PARANOID
18389  else
18390  {
18391  std::ostringstream error_stream;
18392  error_stream << "The only one mesh in the problem is not an "
18393  "unstructured mesh,\n"
18394  << "but the flag 'are_there_unstructures_meshes' ("
18395  << are_there_unstructured_meshes
18396  << ") was turned on,\n"
18397  << "this is weird. Please check for any condition "
18398  "that may have\n"
18399  << "turned on this flag!!!!\n\n";
18400  throw OomphLibError(error_stream.str(),
18401  "Problem::load_balance()",
18402  OOMPH_EXCEPTION_LOCATION);
18403  }
18404 #endif
18405 
18406  unstructured_mesh_pt[0]->load_balance(
18407  target_domain_for_local_non_halo_element);
18408  } // if (n_mesh == 0)
18409  else
18410  {
18411  // Before doing the load balancing delete the meshes created
18412  // at calling build_mesh(), and restore the pointer to the
18413  // old meshes
18414  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
18415  {
18416  if (is_unstructured_mesh[i_mesh])
18417  {
18418  // Delete the new created mesh
18419  delete mesh_pt(i_mesh);
18420  // Now point it to nothing
18421  mesh_pt(i_mesh) = 0;
18422  // ... and re-assign the pointer to the old mesh
18423  this->mesh_pt(i_mesh) = old_mesh_pt[i_mesh];
18424  } // if (is_unstructured_mesh[i_mesh])
18425 
18426  } // for (i_mesh<n_mesh)
18427 
18428  // Empty storage for sub-meshes
18429  // flush_sub_meshes();
18430 
18431  // Flush the storage for nodes and elements in compound mesh
18432  // (they've already been deleted in the sub-meshes)
18434 
18435  // Now we can procede with the load balancing thing
18436  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
18437  {
18438  if (is_unstructured_mesh[i_mesh])
18439  {
18440  // Get the number of elements in the "i_mesh" (the old one)
18441  const unsigned n_element = old_mesh_pt[i_mesh]->nelement();
18442 
18443  // Perform the load balancing if there are elements in the
18444  // mesh. We check for this case because the meshes created
18445  // from face elements have been cleaned in
18446  // "actions_before_distribute()"
18447  if (n_element > 0 && is_unstructured_mesh[i_mesh])
18448  {
18449  unstructured_mesh_pt[i_mesh]->load_balance(
18450  target_domain_for_local_non_halo_element_submesh[i_mesh]);
18451  } // if (n_element > 0)
18452  } // if (is_unstructured_mesh[i_mesh)]
18453  } // for (i_mesh < n_mesh)
18454 
18455  // Rebuild the global mesh
18457 
18458  } // else if (n_mesh == 0)
18459 
18460  } // if (are_there_unstructured_meshes)
18461 
18462  if (report_stats)
18463  {
18464  t_copy_solution = TimingHelpers::timer();
18465  oomph_info << "CPU for transferring solution to new mesh(es): "
18466  << t_copy_solution - t_refine << std::endl;
18467  oomph_info << "CPU for load balancing: " << t_copy_solution - t_start
18468  << std::endl;
18469  }
18470 
18471  // Do actions after distribution
18473 
18474  // Re-assign equation numbers
18475 #ifdef PARANOID
18476  unsigned n_dof = assign_eqn_numbers();
18477 #else
18479 #endif
18480 
18481  if (report_stats)
18482  {
18483  oomph_info
18484  << "Total number of elements on this processor after load balance: "
18485  << mesh_pt()->nelement() << std::endl;
18486 
18487  oomph_info << "Number of non-halo elements on this processor after "
18488  "load balance: "
18489  << mesh_pt()->nnon_halo_element() << std::endl;
18490  }
18491 
18492 #ifdef PARANOID
18493  if (n_dof != old_ndof)
18494  {
18495  std::ostringstream error_stream;
18496  error_stream
18497  << "Number of dofs in load_balance() has changed from " << old_ndof
18498  << " to " << n_dof << "\n"
18499  << "Check that you've implemented any necessary "
18500  "actions_before/after\n"
18501  << "adapt/distribute functions, e.g. to pin redundant pressure dofs"
18502  << " etc.\n";
18503  throw OomphLibError(
18504  error_stream.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
18505  }
18506 #endif
18507  }
18508 
18509  // Finally synchronise all dofs to allow halo check to pass
18511 
18512  double end_t = TimingHelpers::timer();
18513  oomph_info << "Time for load_balance() [sec] : " << end_t - start_t
18514  << std::endl;
18515  }
18516 
18517 
18518  //==========================================================================
18519  /// Send refinement information between processors
18520  //==========================================================================
18522  Vector<unsigned>& old_domain_for_base_element,
18523  Vector<unsigned>& new_domain_for_base_element,
18524  const unsigned& max_refinement_level_overall,
18525  std::map<unsigned, Vector<unsigned>>& flat_packed_refinement_info_for_root,
18526  Vector<Vector<Vector<unsigned>>>& refinement_info_for_root_elements)
18527  {
18528  // Number of processes etc.
18529  const int n_proc = this->communicator_pt()->nproc();
18530  const int my_rank = this->communicator_pt()->my_rank();
18531 
18532  // Make space
18533  unsigned n_base_element = old_domain_for_base_element.size();
18534  refinement_info_for_root_elements.resize(n_base_element);
18535 
18536  // Make space for list of domains that the refinement info
18537  // is to be forwarded to
18538  std::map<unsigned, Vector<unsigned>> halo_domain_of_haloed_base_element;
18539 
18540  // Find out haloed elements in new, redistributed problem
18541  //-------------------------------------------------------
18542 
18543  // halo_domains[e][j] = j-th halo domain associated with (haloed) element e
18544  std::map<unsigned, Vector<unsigned>> halo_domains;
18545 
18546  // Loop over sub meshes
18547  unsigned n_sub_mesh = nsub_mesh();
18548  unsigned max_mesh = std::max(n_sub_mesh, unsigned(1));
18549  for (unsigned i_mesh = 0; i_mesh < max_mesh; i_mesh++)
18550  {
18551  // Choose the right mesh
18552  Mesh* my_mesh_pt = 0;
18553  if (n_sub_mesh == 0)
18554  {
18555  my_mesh_pt = mesh_pt();
18556  }
18557  else
18558  {
18559  my_mesh_pt = mesh_pt(i_mesh);
18560  }
18561 
18562  // Only work with structured meshes
18563  TriangleMeshBase* sub_mesh_pt =
18564  dynamic_cast<TriangleMeshBase*>(mesh_pt(i_mesh));
18565  if (!(sub_mesh_pt != 0))
18566  {
18567  // Loop over processors to find haloed elements -- need to
18568  // send their refinement patterns processors that hold their
18569  // halo counterparts!
18570  for (int p = 0; p < n_proc; p++)
18571  {
18572  Vector<GeneralisedElement*> haloed_elem_pt =
18573  my_mesh_pt->haloed_element_pt(p);
18574  unsigned nhaloed = haloed_elem_pt.size();
18575  for (unsigned h = 0; h < nhaloed; h++)
18576  {
18577  // This element must send its refinement information to processor p
18578  unsigned e = Base_mesh_element_number_plus_one[haloed_elem_pt[h]];
18579 #ifdef PARANOID
18580  if (e == 0)
18581  {
18582  throw OomphLibError("Base_mesh_element_number_plus_one[...]=0",
18583  OOMPH_CURRENT_FUNCTION,
18584  OOMPH_EXCEPTION_LOCATION);
18585  }
18586 #endif
18587  e -= 1;
18588  halo_domains[e].push_back(p);
18589  }
18590  }
18591  } // if (!(sub_mesh_pt!=0))
18592  } // for (i_mesh<max_mesh)
18593 
18594  // Accumulate relevant flat-packed refinement data to be sent to
18595  //--------------------------------------------------------------
18596  // various processors
18597  //-------------------
18598 
18599  // Map to accumulate unsigned data to be sent to each processor
18600  // (map for sparsity)
18601  std::map<unsigned, Vector<unsigned>> data_for_proc;
18602 
18603  // Number of base elements to be sent to specified domain
18604  Vector<unsigned> nbase_elements_for_proc(n_proc, 0);
18605 
18606  // Total number of entries in send vector
18607  unsigned count = 0;
18608 
18609  // Loop over all base elements
18610  //----------------------------
18611  for (unsigned e = 0; e < n_base_element; e++)
18612  {
18613  // Is it one of mine (i.e. was it a non-halo element on this
18614  //----------------------------------------------------------
18615  // processor before re-distribution, and do I therefore hold
18616  //----------------------------------------------------------
18617  // refinement information for it)?
18618  //--------------------------------
18619  if (int(old_domain_for_base_element[e]) == my_rank)
18620  {
18621  // Where does it go?
18622  unsigned new_domain = new_domain_for_base_element[e];
18623 
18624  // Keep counting
18625  nbase_elements_for_proc[new_domain]++;
18626 
18627  // If it stays local, deal with it here
18628  if (int(new_domain) == my_rank)
18629  {
18630  // Record on which other procs/domains the refinement info for
18631  // this element is required because it's haloed.
18632  unsigned nhalo = halo_domains[e].size();
18633  halo_domain_of_haloed_base_element[e].resize(nhalo);
18634  for (unsigned j = 0; j < nhalo; j++)
18635  {
18636  halo_domain_of_haloed_base_element[e][j] = halo_domains[e][j];
18637  }
18638 
18639  // Provide storage for refinement pattern
18640  refinement_info_for_root_elements[e].resize(
18641  max_refinement_level_overall);
18642 
18643 #ifdef PARANOID
18644  // Get number of additional data sent for check
18645  unsigned n_additional_data =
18646  flat_packed_refinement_info_for_root[e].size();
18647 #endif
18648 
18649  // Get number of tree nodes
18650  unsigned n_tree_nodes = flat_packed_refinement_info_for_root[e][0];
18651 
18652  // Counter for entries to be processed locally
18653  unsigned local_count = 1; // (have already processed zero-th entry)
18654 
18655  // Loop over levels and number of nodes in tree
18656  for (unsigned level = 0; level < max_refinement_level_overall;
18657  level++)
18658  {
18659  for (unsigned ee = 0; ee < n_tree_nodes; ee++)
18660  {
18661  // Element exists at this level
18662  if (flat_packed_refinement_info_for_root[e][local_count] == 1)
18663  {
18664  local_count++;
18665 
18666  // Element should be refined
18667  if (flat_packed_refinement_info_for_root[e][local_count] == 1)
18668  {
18669  refinement_info_for_root_elements[e][level].push_back(2);
18670  local_count++;
18671  }
18672  // Element should not be refined
18673  else
18674  {
18675  refinement_info_for_root_elements[e][level].push_back(1);
18676  local_count++;
18677  }
18678  }
18679  // Element does not exist at this level
18680  else
18681  {
18682  refinement_info_for_root_elements[e][level].push_back(0);
18683  local_count++;
18684  }
18685  }
18686  }
18687 
18688 #ifdef PARANOID
18689  if (n_additional_data != local_count)
18690  {
18691  std::stringstream error_message;
18692  error_message << "Number of additional data: " << n_additional_data
18693  << " doesn't match that actually send: "
18694  << local_count << std::endl;
18695  throw OomphLibError(error_message.str(),
18696  OOMPH_CURRENT_FUNCTION,
18697  OOMPH_EXCEPTION_LOCATION);
18698  }
18699 #endif
18700  }
18701  // Element in question is not one of mine so prepare for sending
18702  //--------------------------------------------------------------
18703  else
18704  {
18705  // Make space
18706  unsigned current_size = data_for_proc[new_domain].size();
18707  unsigned n_additional_data =
18708  flat_packed_refinement_info_for_root[e].size();
18709  data_for_proc[new_domain].reserve(current_size + n_additional_data +
18710  2);
18711 
18712  // Keep counting
18713  count += n_additional_data + 2;
18714 
18715  // Add base element number
18716  data_for_proc[new_domain].push_back(e);
18717 
18718 #ifdef PARANOID
18719  // Add number of flat-packed instructions to follow
18720  data_for_proc[new_domain].push_back(n_additional_data);
18721 #endif
18722 
18723  // Add flat packed refinement data
18724  for (unsigned j = 0; j < n_additional_data; j++)
18725  {
18726  data_for_proc[new_domain].push_back(
18727  flat_packed_refinement_info_for_root[e][j]);
18728  }
18729  }
18730  }
18731  }
18732 
18733 
18734  // Now do the actual send/receive
18735  //-------------------------------
18736 
18737  // Storage for number of data to be sent to each processor
18738  Vector<int> send_n(n_proc, 0);
18739 
18740  // Storage for all values to be sent to all processors
18741  Vector<unsigned> send_data;
18742  send_data.reserve(count);
18743 
18744  // Start location within send_data for data to be sent to each processor
18745  Vector<int> send_displacement(n_proc, 0);
18746 
18747  // Loop over all processors
18748  for (int rank = 0; rank < n_proc; rank++)
18749  {
18750  // Set the offset for the current processor
18751  send_displacement[rank] = send_data.size();
18752 
18753  // Don't bother to do anything if the processor in the loop is the
18754  // current processor
18755  if (rank != my_rank)
18756  {
18757  // Record how many base elements are to be sent
18758  send_data.push_back(nbase_elements_for_proc[rank]);
18759 
18760  // Add data
18761  unsigned n_data = data_for_proc[rank].size();
18762  for (unsigned j = 0; j < n_data; j++)
18763  {
18764  send_data.push_back(data_for_proc[rank][j]);
18765  }
18766  }
18767 
18768  // Find the number of data added to the vector
18769  send_n[rank] = send_data.size() - send_displacement[rank];
18770  }
18771 
18772  // Storage for the number of data to be received from each processor
18773  Vector<int> receive_n(n_proc, 0);
18774 
18775  // Now send numbers of data to be sent between all processors
18776  MPI_Alltoall(&send_n[0],
18777  1,
18778  MPI_INT,
18779  &receive_n[0],
18780  1,
18781  MPI_INT,
18782  this->communicator_pt()->mpi_comm());
18783 
18784  // We now prepare the data to be received
18785  // by working out the displacements from the received data
18786  Vector<int> receive_displacement(n_proc, 0);
18787  int receive_data_count = 0;
18788  for (int rank = 0; rank < n_proc; ++rank)
18789  {
18790  // Displacement is number of data received so far
18791  receive_displacement[rank] = receive_data_count;
18792  receive_data_count += receive_n[rank];
18793  }
18794 
18795  // Now resize the receive buffer for all data from all processors
18796  // Make sure that it has a size of at least one
18797  if (receive_data_count == 0)
18798  {
18799  ++receive_data_count;
18800  }
18801  Vector<unsigned> receive_data(receive_data_count);
18802 
18803  // Make sure that the send buffer has size at least one
18804  // so that we don't get a segmentation fault
18805  if (send_data.size() == 0)
18806  {
18807  send_data.resize(1);
18808  }
18809 
18810  // Now send the data between all the processors
18811  MPI_Alltoallv(&send_data[0],
18812  &send_n[0],
18813  &send_displacement[0],
18814  MPI_UNSIGNED,
18815  &receive_data[0],
18816  &receive_n[0],
18817  &receive_displacement[0],
18818  MPI_UNSIGNED,
18819  this->communicator_pt()->mpi_comm());
18820 
18821 
18822  // Now use the received data to update
18823  //-----------------------------------
18824  for (int send_rank = 0; send_rank < n_proc; send_rank++)
18825  {
18826  // Don't bother to do anything for the processor corresponding to the
18827  // current processor or if no data were received from this processor
18828  if ((send_rank != my_rank) && (receive_n[send_rank] != 0))
18829  {
18830  // Counter for the data within the large array
18831  unsigned count = receive_displacement[send_rank];
18832 
18833  // Loop over base elements
18834  unsigned nbase_element = receive_data[count];
18835  count++;
18836  for (unsigned b = 0; b < nbase_element; b++)
18837  {
18838  // Get base element number
18839  unsigned base_element_number = receive_data[count];
18840  count++;
18841 
18842  // Record on which other procs/domains the refinement info for
18843  // this element is required because it's haloed.
18844  unsigned nhalo = halo_domains[base_element_number].size();
18845  halo_domain_of_haloed_base_element[base_element_number].resize(nhalo);
18846  for (unsigned j = 0; j < nhalo; j++)
18847  {
18848  halo_domain_of_haloed_base_element[base_element_number][j] =
18849  halo_domains[base_element_number][j];
18850  }
18851 
18852  // Provide storage for refinement pattern
18853  refinement_info_for_root_elements[base_element_number].resize(
18854  max_refinement_level_overall);
18855 
18856  // Get number of flat-packed instructions to follow
18857  // (only used for check)
18858 #ifdef PARANOID
18859  unsigned n_additional_data = receive_data[count];
18860  count++;
18861 
18862  // Counter for number of additional data (validation only)
18863  unsigned check_count = 0;
18864 #endif
18865 
18866  // Get number of tree nodes
18867  unsigned n_tree_nodes = receive_data[count];
18868  count++;
18869 
18870 #ifdef PARANOID
18871  check_count++;
18872 #endif
18873 
18874  // Loop over levels and number of nodes in tree
18875  for (unsigned level = 0; level < max_refinement_level_overall;
18876  level++)
18877  {
18878  for (unsigned e = 0; e < n_tree_nodes; e++)
18879  {
18880  // Element exists at this level
18881  if (receive_data[count] == 1)
18882  {
18883  count++;
18884 
18885 #ifdef PARANOID
18886  check_count++;
18887 #endif
18888 
18889  // Element should be refined
18890  if (receive_data[count] == 1)
18891  {
18892  refinement_info_for_root_elements[base_element_number][level]
18893  .push_back(2);
18894  count++;
18895 
18896 #ifdef PARANOID
18897  check_count++;
18898 #endif
18899  }
18900  // Element should not be refined
18901  else
18902  {
18903  refinement_info_for_root_elements[base_element_number][level]
18904  .push_back(1);
18905  count++;
18906 
18907 #ifdef PARANOID
18908  check_count++;
18909 #endif
18910  }
18911  }
18912  // Element does not exist at this level
18913  else
18914  {
18915  refinement_info_for_root_elements[base_element_number][level]
18916  .push_back(0);
18917  count++;
18918 
18919 #ifdef PARANOID
18920  check_count++;
18921 #endif
18922  }
18923  }
18924  }
18925 
18926 #ifdef PARANOID
18927  if (n_additional_data != check_count)
18928  {
18929  std::stringstream error_message;
18930  error_message << "Number of additional data: " << n_additional_data
18931  << " doesn't match that actually send: "
18932  << check_count << std::endl;
18933  throw OomphLibError(error_message.str(),
18934  OOMPH_CURRENT_FUNCTION,
18935  OOMPH_EXCEPTION_LOCATION);
18936  }
18937 #endif
18938  }
18939  }
18940  }
18941 
18942 
18943  // Now send the fully assembled refinement info to halo elements
18944  //---------------------------------------------------------------
18945  {
18946  // Accumulate data to be sent
18947  //---------------------------
18948 
18949  // Map to accumulate data to be sent to other procs
18950  // (map for sparsity)
18951  std::map<unsigned, Vector<unsigned>> data_for_proc;
18952 
18953  // Number of base elements to be sent to specified domain
18954  Vector<unsigned> nbase_elements_for_proc(n_proc, 0);
18955 
18956  // Loop over all haloed root elements and find out which
18957  // processors they have haloes on
18958  for (std::map<unsigned, Vector<unsigned>>::iterator it =
18959  halo_domain_of_haloed_base_element.begin();
18960  it != halo_domain_of_haloed_base_element.end();
18961  it++)
18962  {
18963  // Get base element number
18964  unsigned base_element_number = (*it).first;
18965 
18966  // Loop over target domains
18967  Vector<unsigned> domains = (*it).second;
18968  unsigned nd = domains.size();
18969  for (unsigned jd = 0; jd < nd; jd++)
18970  {
18971  // Actual number of domain
18972  unsigned d = domains[jd];
18973 
18974  // Keep counting number of base elemements for domain
18975  nbase_elements_for_proc[d]++;
18976 
18977  // Write base element number
18978  data_for_proc[d].push_back(base_element_number);
18979 
18980  // Write refinement info in flat-packed form
18981  for (unsigned level = 0; level < max_refinement_level_overall;
18982  level++)
18983  {
18984  // Number of entries at each level
18985  unsigned n =
18986  refinement_info_for_root_elements[base_element_number][level]
18987  .size();
18988  data_for_proc[d].push_back(n);
18989  for (unsigned j = 0; j < n; j++)
18990  {
18991  data_for_proc[d].push_back(
18992  refinement_info_for_root_elements[base_element_number][level]
18993  [j]);
18994  }
18995  }
18996  }
18997  }
18998 
18999 
19000  // Do the actual send
19001  //-------------------
19002 
19003  // Storage for number of data to be sent to each processor
19004  Vector<int> send_n(n_proc, 0);
19005 
19006  // Storage for all values to be sent to all processors
19007  Vector<unsigned> send_data;
19008  send_data.reserve(count);
19009 
19010  // Start location within send_data for data to be sent to each processor
19011  Vector<int> send_displacement(n_proc, 0);
19012 
19013  // Loop over all processors
19014  for (int rank = 0; rank < n_proc; rank++)
19015  {
19016  // Set the offset for the current processor
19017  send_displacement[rank] = send_data.size();
19018 
19019  // Don't bother to do anything if the processor in the loop is the
19020  // current processor
19021  if (rank != my_rank)
19022  {
19023  // Record how many base elements are to be sent
19024  send_data.push_back(nbase_elements_for_proc[rank]);
19025 
19026  // Add data
19027  unsigned n_data = data_for_proc[rank].size();
19028  for (unsigned j = 0; j < n_data; j++)
19029  {
19030  send_data.push_back(data_for_proc[rank][j]);
19031  }
19032  }
19033  // Find the number of data added to the vector
19034  send_n[rank] = send_data.size() - send_displacement[rank];
19035  }
19036 
19037  // Storage for the number of data to be received from each processor
19038  Vector<int> receive_n(n_proc, 0);
19039 
19040  // Now send numbers of data to be sent between all processors
19041  MPI_Alltoall(&send_n[0],
19042  1,
19043  MPI_INT,
19044  &receive_n[0],
19045  1,
19046  MPI_INT,
19047  this->communicator_pt()->mpi_comm());
19048 
19049  // We now prepare the data to be received
19050  // by working out the displacements from the received data
19051  Vector<int> receive_displacement(n_proc, 0);
19052  int receive_data_count = 0;
19053  for (int rank = 0; rank < n_proc; ++rank)
19054  {
19055  // Displacement is number of data received so far
19056  receive_displacement[rank] = receive_data_count;
19057  receive_data_count += receive_n[rank];
19058  }
19059 
19060  // Now resize the receive buffer for all data from all processors
19061  // Make sure that it has a size of at least one
19062  if (receive_data_count == 0)
19063  {
19064  ++receive_data_count;
19065  }
19066  Vector<unsigned> receive_data(receive_data_count);
19067 
19068  // Make sure that the send buffer has size at least one
19069  // so that we don't get a segmentation fault
19070  if (send_data.size() == 0)
19071  {
19072  send_data.resize(1);
19073  }
19074 
19075  // Now send the data between all the processors
19076  MPI_Alltoallv(&send_data[0],
19077  &send_n[0],
19078  &send_displacement[0],
19079  MPI_UNSIGNED,
19080  &receive_data[0],
19081  &receive_n[0],
19082  &receive_displacement[0],
19083  MPI_UNSIGNED,
19084  this->communicator_pt()->mpi_comm());
19085 
19086 
19087  // Now use the received data
19088  //------------------------
19089  for (int send_rank = 0; send_rank < n_proc; send_rank++)
19090  {
19091  // Don't bother to do anything for the processor corresponding to the
19092  // current processor or if no data were received from this processor
19093  if ((send_rank != my_rank) && (receive_n[send_rank] != 0))
19094  {
19095  // Counter for the data within the large array
19096  unsigned count = receive_displacement[send_rank];
19097 
19098  // Read number of base elements
19099  unsigned nbase_element = receive_data[count];
19100  count++;
19101 
19102  for (unsigned e = 0; e < nbase_element; e++)
19103  {
19104  // Read base element number
19105  unsigned base_element_number = receive_data[count];
19106  count++;
19107 
19108  // Provide storage for refinement pattern
19109  refinement_info_for_root_elements[base_element_number].resize(
19110  max_refinement_level_overall);
19111 
19112  // Read refinement info in flat-packed form
19113  for (unsigned level = 0; level < max_refinement_level_overall;
19114  level++)
19115  {
19116  // Read number of entries at each level
19117  unsigned n = receive_data[count];
19118  count++;
19119 
19120  // Read entries
19121  for (unsigned j = 0; j < n; j++)
19122  {
19123  refinement_info_for_root_elements[base_element_number][level]
19124  .push_back(receive_data[count]);
19125  count++;
19126  }
19127  }
19128  }
19129  }
19130  }
19131  }
19132  }
19133 
19134  //==========================================================================
19135  /// Load balance helper routine: Send data to other
19136  /// processors during load balancing.
19137  /// - send_n: Input, number of data to be sent to each processor
19138  /// - send_data: Input, storage for all values to be sent to all processors
19139  /// - send_displacement: Input, start location within send_data for data to
19140  /// be sent to each processor
19141  //==========================================================================
19143  Vector<int>& send_n,
19144  Vector<double>& send_data,
19145  Vector<int>& send_displacement)
19146  {
19147  // Communicator info
19148  OomphCommunicator* comm_pt = this->communicator_pt();
19149  const int n_proc = comm_pt->nproc();
19150 
19151  // Storage for the number of data to be received from each processor
19152  Vector<int> receive_n(n_proc, 0);
19153 
19154  // Now send numbers of data to be sent between all processors
19155  MPI_Alltoall(&send_n[0],
19156  1,
19157  MPI_INT,
19158  &receive_n[0],
19159  1,
19160  MPI_INT,
19161  this->communicator_pt()->mpi_comm());
19162 
19163  // We now prepare the data to be received
19164  // by working out the displacements from the received data
19165  Vector<int> receive_displacement(n_proc, 0);
19166  int receive_data_count = 0;
19167  for (int rank = 0; rank < n_proc; ++rank)
19168  {
19169  // Displacement is number of data received so far
19170  receive_displacement[rank] = receive_data_count;
19171  receive_data_count += receive_n[rank];
19172  }
19173 
19174  // Now resize the receive buffer for all data from all processors
19175  // Make sure that it has a size of at least one
19176  if (receive_data_count == 0)
19177  {
19178  ++receive_data_count;
19179  }
19180  Vector<double> receive_data(receive_data_count);
19181 
19182  // Make sure that the send buffer has size at least one
19183  // so that we don't get a segmentation fault
19184  if (send_data.size() == 0)
19185  {
19186  send_data.resize(1);
19187  }
19188 
19189  // Now send the data between all the processors
19190  MPI_Alltoallv(&send_data[0],
19191  &send_n[0],
19192  &send_displacement[0],
19193  MPI_DOUBLE,
19194  &receive_data[0],
19195  &receive_n[0],
19196  &receive_displacement[0],
19197  MPI_DOUBLE,
19198  this->communicator_pt()->mpi_comm());
19199 
19200  unsigned el_count = 0;
19201 
19202  // Only do each node once
19203  Vector<std::map<Node*, bool>> node_done(n_proc);
19204 
19205  // Now use the received data to update the halo nodes
19206  for (int send_rank = 0; send_rank < n_proc; send_rank++)
19207  {
19208  // Don't bother to do anything if no data were received from this
19209  // processor
19210  // NOTE: We do have to loop over our own processor number to process
19211  // the data locally.
19212  if (receive_n[send_rank] != 0)
19213  {
19214  // Counter for the data within the large array
19215  unsigned count = receive_displacement[send_rank];
19216 
19217  // How many batches are there for current rank
19218  unsigned nbatch = unsigned(receive_data[count]);
19219  count++;
19220 
19221  // Loop over batches (containing leaves associated with root elements)
19222  for (unsigned b = 0; b < nbatch; b++)
19223  {
19224  // How many elements were received for this batch?
19225  unsigned nel = unsigned(receive_data[count]);
19226  count++;
19227 
19228  // Get the unique base/root element number of this batch
19229  // in unrefined mesh
19230  unsigned base_el_no = unsigned(receive_data[count]);
19231  count++;
19232 
19233  // Get pointer to base/root element from reverse lookup scheme
19234  GeneralisedElement* root_el_pt = Base_mesh_element_pt[base_el_no];
19235 
19236  // Vector for pointers to associated elements in batch
19237  Vector<GeneralisedElement*> batch_el_pt;
19238 
19239  // Is it a refineable element?
19240  RefineableElement* ref_root_el_pt =
19241  dynamic_cast<RefineableElement*>(root_el_pt);
19242  if (ref_root_el_pt != 0)
19243  {
19244  // Get all leaves associated with this base/root element
19245  Vector<Tree*> all_leaf_nodes_pt;
19246  ref_root_el_pt->tree_pt()->stick_leaves_into_vector(
19247  all_leaf_nodes_pt);
19248 
19249  // How many leaves are there?
19250  unsigned n_leaf = all_leaf_nodes_pt.size();
19251 
19252 #ifdef PARANOID
19253  if (n_leaf != nel)
19254  {
19255  std::ostringstream error_message;
19256  error_message
19257  << "Number of leaves: " << n_leaf << " "
19258  << " doesn't match number of elements sent in batch: " << nel
19259  << "\n";
19260  throw OomphLibError(error_message.str(),
19261  OOMPH_CURRENT_FUNCTION,
19262  OOMPH_EXCEPTION_LOCATION);
19263  }
19264 #endif
19265 
19266  // Loop over batch of elements associated with this base/root
19267  // element
19268  batch_el_pt.resize(n_leaf);
19269  for (unsigned e = 0; e < n_leaf; e++)
19270  {
19271  batch_el_pt[e] = all_leaf_nodes_pt[e]->object_pt();
19272  }
19273  }
19274  // Not refineable -- the batch contains just the root element itself
19275  else
19276  {
19277 #ifdef PARANOID
19278  if (1 != nel)
19279  {
19280  std::ostringstream error_message;
19281  error_message
19282  << "Non-refineable root element should only be associated with"
19283  << " one element but nel=" << nel << "\n";
19284  throw OomphLibError(error_message.str(),
19285  OOMPH_CURRENT_FUNCTION,
19286  OOMPH_EXCEPTION_LOCATION);
19287  }
19288 #endif
19289  batch_el_pt.push_back(root_el_pt);
19290  }
19291 
19292  // Now loop over all elements in batch
19293  for (unsigned e = 0; e < nel; e++)
19294  {
19295  GeneralisedElement* el_pt = batch_el_pt[e];
19296  el_count++;
19297 
19298  // FE?
19299  FiniteElement* fe_pt = dynamic_cast<FiniteElement*>(el_pt);
19300  if (fe_pt != 0)
19301  {
19302  // Loop over nodes
19303  unsigned nnod = fe_pt->nnode();
19304  for (unsigned j = 0; j < nnod; j++)
19305  {
19306  Node* nod_pt = fe_pt->node_pt(j);
19307  if (!node_done[send_rank][nod_pt])
19308  {
19309  node_done[send_rank][nod_pt] = true;
19310 
19311 
19312  // Read number of values (as double) to allow for resizing
19313  // before read (req'd in case we store data that
19314  // got introduced by attaching FaceElements to bulk)
19315  unsigned nval = unsigned(receive_data[count]);
19316  count++;
19317 
19318 #ifdef PARANOID
19319  // Does the size match?
19320  if (nval < nod_pt->nvalue())
19321  {
19322  std::ostringstream error_message;
19323  error_message
19324  << "Node has more values, namely " << nod_pt->nvalue()
19325  << ", than we're about to receive, namely " << nval
19326  << ". Something's wrong!\n";
19327  throw OomphLibError(error_message.str(),
19328  OOMPH_CURRENT_FUNCTION,
19329  OOMPH_EXCEPTION_LOCATION);
19330  }
19331 #endif
19332 
19333 
19334 #ifdef PARANOID
19335  // Check if it's been sent as a boundary node
19336  unsigned is_boundary_node = unsigned(receive_data[count]);
19337  count++;
19338 #endif
19339 
19340  // Check if it's actually a boundary node
19341  BoundaryNodeBase* bnod_pt =
19342  dynamic_cast<BoundaryNodeBase*>(nod_pt);
19343  if (bnod_pt != 0)
19344  {
19345 #ifdef PARANOID
19346  // Check if local and received status are consistent
19347  if (is_boundary_node != 1)
19348  {
19349  std::ostringstream error_message;
19350  error_message << "Local node is boundary node but "
19351  "information sent is\n"
19352  << "for non-boundary node\n";
19353  throw OomphLibError(error_message.str(),
19354  OOMPH_CURRENT_FUNCTION,
19355  OOMPH_EXCEPTION_LOCATION);
19356  }
19357 #endif
19358 
19359  // Do we have entries in the map?
19360  unsigned n_entry = unsigned(receive_data[count]);
19361  count++;
19362  if (n_entry > 0)
19363  {
19364  // Create storage, if it doesn't already exist, for the
19365  // map that will contain the position of the first entry
19366  // of this face element's additional values,
19367  if (
19368  bnod_pt
19369  ->index_of_first_value_assigned_by_face_element_pt() ==
19370  0)
19371  {
19372  bnod_pt
19374  new std::map<unsigned, unsigned>;
19375  }
19376 
19377  // Get pointer to the map of indices associated with
19378  // additional values created by face elements
19379  std::map<unsigned, unsigned>* map_pt =
19380  bnod_pt
19382 
19383  // Loop over number of entries in map
19384  for (unsigned i = 0; i < n_entry; i++)
19385  {
19386  // Read out pairs...
19387  unsigned first = unsigned(receive_data[count]);
19388  count++;
19389  unsigned second = unsigned(receive_data[count]);
19390  count++;
19391 
19392  // ...and assign
19393  (*map_pt)[first] = second;
19394  }
19395  }
19396  }
19397 #ifdef PARANOID
19398  // Not a boundary node
19399  else
19400  {
19401  // Check if local and received status are consistent
19402  if (is_boundary_node != 0)
19403  {
19404  std::ostringstream error_message;
19405  error_message << "Local node is not a boundary node but "
19406  "information \n"
19407  << "sent is for boundary node.\n";
19408  throw OomphLibError(error_message.str(),
19409  OOMPH_CURRENT_FUNCTION,
19410  OOMPH_EXCEPTION_LOCATION);
19411  }
19412  }
19413 #endif
19414 
19415  // Do we have to resize? This can happen if node was
19416  // resized (due to a FaceElement that hasn't been attached
19417  // yet here) when the send data was written. If so make space
19418  // for the data here
19419  if (nval > nod_pt->nvalue())
19420  {
19421  nod_pt->resize(nval);
19422  }
19423 
19424  // Now read the actual values
19425  nod_pt->read_values_from_vector(receive_data, count);
19426  }
19427  }
19428  }
19429 
19430  // Now add internal data
19431  el_pt->read_internal_data_values_from_vector(receive_data, count);
19432  }
19433  }
19434  }
19435  }
19436 
19437  // Now that this is done, we need to synchronise dofs to get
19438  // the halo element and node values correct
19439  bool do_halos = true;
19440  bool do_external_halos = false;
19441  this->synchronise_dofs(do_halos, do_external_halos);
19442 
19443  // Now rebuild global mesh if required
19444  unsigned n_mesh = nsub_mesh();
19445  if (n_mesh != 0)
19446  {
19447  bool do_halos = false;
19448  bool do_external_halos = true;
19449  this->synchronise_dofs(do_halos, do_external_halos);
19451  }
19452  }
19453 
19454 
19455  //==========================================================================
19456  /// Load balance helper routine: Get data to be sent to other
19457  /// processors during load balancing and other information about
19458  /// re-distribution.
19459  /// - target_domain_for_local_non_halo_element: Input, generated by METIS.
19460  /// target_domain_for_local_non_halo_element[e] contains the number
19461  /// of the domain [0,1,...,nproc-1] to which non-halo element e on THE
19462  /// CURRENT PROCESSOR ONLY has been assigned. The order of the non-halo
19463  /// elements is the same as in the Problem's mesh, with the halo
19464  /// elements being skipped.
19465  /// - send_n: Output, number of data to be sent to each processor
19466  /// - send_data: Output, storage for all values to be sent to all processors
19467  /// - send_displacement: Output, start location within send_data for data to
19468  /// be sent to each processor
19469  /// - max_refinement_level_overall: Output, max. refinement level of any
19470  /// element
19471  //==========================================================================
19473  const Vector<unsigned>& target_domain_for_local_non_halo_element,
19474  Vector<int>& send_n,
19475  Vector<double>& send_data,
19476  Vector<int>& send_displacement,
19477  Vector<unsigned>& old_domain_for_base_element,
19478  Vector<unsigned>& new_domain_for_base_element,
19479  unsigned& max_refinement_level_overall)
19480  {
19481  // Communicator info
19482  OomphCommunicator* comm_pt = this->communicator_pt();
19483  const int n_proc = comm_pt->nproc();
19484  const int my_rank = this->communicator_pt()->my_rank();
19485 
19486  //------------------------------------------------------------------------
19487  // Overall strategy: Loop over all elements (in structured meshes),
19488  // identify their corresponding root elements and move all associated
19489  // leaves together, collecting the leaves in batches.
19490  // ------------------------------------------------------------------------
19491 
19492  // Map to store whether the root element has been visited yet
19493  std::map<RefineableElement*, bool> root_el_done;
19494 
19495 #ifdef PARANOID
19496 
19497  // Map for checking if all elements associated with same root
19498  // have the same target processor
19499  std::map<RefineableElement*, unsigned> target_plus_one_for_root;
19500 
19501 #endif
19502 
19503  // Storage for maximum refinement level
19504  unsigned max_refinement_level = 0;
19505 
19506  // Storage for (vector of) elements associated with target domain
19507  // (stored in map for sparsity): element_for_processor[d][e] is pointer
19508  // to e-th element that's supposed to move onto processor (domain) d.
19509  std::map<unsigned, Vector<GeneralisedElement*>> element_for_processor;
19510 
19511  // Storage for the number of elements in a specified batch of leaf
19512  // elements, all of which are associated with the same root/base element:
19513  // nelement_batch_for_processor[d][j] is the number of (leaf)
19514  // elements (all associated with the same root) to be moved together to
19515  // domain/processor d, in the j-th batch of elements.
19516  std::map<unsigned, Vector<unsigned>> nelement_batch_for_processor;
19517 
19518  // Storage for the unique number of the root element (in the unrefined
19519  // base mesh) whose leaves are moved together in a batch:
19520  // base_element_for_element_batch_for_processo[d][j] is the number of
19521  // unique number of the root element (in the unrefined
19522  // base mesh) of all leaf elements (associated with that root),
19523  // to be moved together to domain/processor d, in the j-th batch of
19524  // elements.
19525  std::map<unsigned, Vector<unsigned>>
19526  base_element_for_element_batch_for_processor;
19527 
19528  // Record old and new domains for non-halo root elements (will be
19529  // communicated globally). Initialise to -1 so we can use max
19530  // to extract the right one via MPI_Allreduce.
19531  // NOTE: We communicate these globally to facilitate distribution
19532  // of refinement pattern. While the data itself can be
19533  // sent point-to-point for non-halo elements,
19534  // mesh refinement information also needs to be sent for
19535  // halo elements which aren't known yet.
19536  unsigned n_base_element = Base_mesh_element_pt.size();
19537  Vector<int> old_domain_for_base_element_local(n_base_element, -1);
19538  Vector<int> new_domain_for_base_element_local(n_base_element, -1);
19539 
19540  // Loop over all non-halo elements on current processor and identify roots
19541  // -------------------------------------------------------------------
19542  // All leaf elements in associated tree (must!) get moved together
19543  //----------------------------------------------------------------
19544  unsigned count_non_halo_el = 0;
19545  // Get the number of submeshs, if there are no submeshes, then
19546  // increase the counter so that the loop below also work for the only
19547  // one mesh in the problem
19548  unsigned n_mesh = nsub_mesh();
19549  if (n_mesh == 0)
19550  {
19551  n_mesh = 1;
19552  }
19553  // We need to know if there are structure meshes (with elements) as
19554  // part of the problem in order to perform (or not) the proper
19555  // communications
19556  bool are_there_structured_meshes = false;
19557  // Go for the nonhalo elements only in the TreeBaseMeshes
19558  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
19559  {
19560  // Only work with structured meshes
19561  TriangleMeshBase* sub_mesh_pt =
19562  dynamic_cast<TriangleMeshBase*>(mesh_pt(i_mesh));
19563  if (!(sub_mesh_pt != 0))
19564  {
19565  const unsigned nele = mesh_pt(i_mesh)->nelement();
19566  if (nele > 0)
19567  {
19568  // Change the flag to indicate that there are structured meshes
19569  // (with elements, because we may have meshes with face
19570  // elements and therefore zero elements at this point)
19571  are_there_structured_meshes = true;
19572  }
19573 
19574  for (unsigned e = 0; e < nele; e++)
19575  {
19576  GeneralisedElement* el_pt = mesh_pt(i_mesh)->element_pt(e);
19577  if (!el_pt->is_halo())
19578  {
19579  // New non-halo: Where is this element supposed to go to?
19580  //-------------------------------------------------------
19581  unsigned target_domain =
19582  target_domain_for_local_non_halo_element[count_non_halo_el];
19583 
19584  // Bump up counter for non-halo elements
19585  count_non_halo_el++;
19586 
19587  // Is it a root element? (It is, trivially, if it's not refineable)
19588  //------------------------------------------------------------------
19589  RefineableElement* ref_el_pt =
19590  dynamic_cast<RefineableElement*>(el_pt);
19591  if (ref_el_pt == 0)
19592  {
19593  // Not refineable so add element itself
19594  element_for_processor[target_domain].push_back(el_pt);
19595 
19596  // Number of elements associated with this root/base
19597  // element (just the element itself)
19598  nelement_batch_for_processor[target_domain].push_back(1);
19599 
19600  // This is the unique base/root element number in unrefined mesh
19601  unsigned element_number_in_base_mesh =
19603 #ifdef PARANOID
19604  if (element_number_in_base_mesh == 0)
19605  {
19606  throw OomphLibError("Base_mesh_element_number_plus_one[...]=0",
19607  OOMPH_CURRENT_FUNCTION,
19608  OOMPH_EXCEPTION_LOCATION);
19609  }
19610 #endif
19611  element_number_in_base_mesh -= 1;
19612  base_element_for_element_batch_for_processor[target_domain]
19613  .push_back(element_number_in_base_mesh);
19614 
19615  /// Where do I come from, where do I go to?
19616  old_domain_for_base_element_local[element_number_in_base_mesh] =
19617  my_rank;
19618  new_domain_for_base_element_local[element_number_in_base_mesh] =
19619  target_domain;
19620  } // if (ref_el_pt==0)
19621  // It's not a root element so we package its leaves into a batch
19622  //--------------------------------------------------------------
19623  // of elements
19624  //------------
19625  else
19626  {
19627  // Get the root element
19628  RefineableElement* root_el_pt = ref_el_pt->root_element_pt();
19629 
19630  // Has this root been visited yet?
19631  if (!root_el_done[root_el_pt])
19632  {
19633  // Now we've done it
19634  root_el_done[root_el_pt] = true;
19635 
19636  // Unique number of root element in base mesh
19637  unsigned element_number_in_base_mesh =
19639 #ifdef PARANOID
19640  if (element_number_in_base_mesh == 0)
19641  {
19642  throw OomphLibError(
19643  "Base_mesh_element_number_plus_one[...]=0",
19644  OOMPH_CURRENT_FUNCTION,
19645  OOMPH_EXCEPTION_LOCATION);
19646  }
19647 #endif
19648  element_number_in_base_mesh -= 1;
19649 
19650  /// Where do I come from, where do I go to?
19651  old_domain_for_base_element_local[element_number_in_base_mesh] =
19652  my_rank;
19653  new_domain_for_base_element_local[element_number_in_base_mesh] =
19654  target_domain;
19655 
19656 #ifdef PARANOID
19657  // Store target domain associated with this root element
19658  // (offset by one) to allow checking that all elements
19659  // with the same root move to the same processor
19660  target_plus_one_for_root[root_el_pt] = target_domain + 1;
19661 #endif
19662 
19663  // Package all leaves into batch of elements
19664  Vector<Tree*> all_leaf_nodes_pt;
19665  root_el_pt->tree_pt()->stick_leaves_into_vector(
19666  all_leaf_nodes_pt);
19667 
19668  // Number of leaves
19669  unsigned n_leaf = all_leaf_nodes_pt.size();
19670 
19671  // Number of elements associated with this root/base element
19672  // (all the leaves)
19673  nelement_batch_for_processor[target_domain].push_back(n_leaf);
19674 
19675  // Store the unique base/root element number in unrefined mesh
19676  base_element_for_element_batch_for_processor[target_domain]
19677  .push_back(element_number_in_base_mesh);
19678 
19679  // Loop over leaves
19680  for (unsigned i_leaf = 0; i_leaf < n_leaf; i_leaf++)
19681  {
19682  // Add element object at leaf
19683  RefineableElement* leaf_el_pt =
19684  all_leaf_nodes_pt[i_leaf]->object_pt();
19685  element_for_processor[target_domain].push_back(leaf_el_pt);
19686 
19687  // Monitor/update maximum refinement level
19688  unsigned level = all_leaf_nodes_pt[i_leaf]->level();
19689  if (level > max_refinement_level)
19690  {
19691  max_refinement_level = level;
19692  }
19693  }
19694  }
19695 
19696 #ifdef PARANOID
19697  // Root element has already been visited
19698  else
19699  {
19700  // We don't have to do anything with this element since it's
19701  // already been processed earlier, but check that it's scheduled
19702  // to go onto the same processor as its root.
19703  if ((target_plus_one_for_root[root_el_pt] - 1) != target_domain)
19704  {
19705  std::ostringstream error_message;
19706  error_message
19707  << "All elements associated with same root must have "
19708  << "same target. during load balancing\n";
19709  throw OomphLibError(error_message.str(),
19710  OOMPH_CURRENT_FUNCTION,
19711  OOMPH_EXCEPTION_LOCATION);
19712  }
19713  }
19714 #endif
19715  } // else if (ref_el_pt==0)
19716  } // if (!ele_pt->is_halo())
19717  } // for (e < nele)
19718  } // if (!(sub_mesh_pt!=0))
19719  } // for (i_mesh < n_mesh)
19720 
19721 #ifdef PARANOID
19722  // Have we processed all target domains?
19723  if (target_domain_for_local_non_halo_element.size() != count_non_halo_el)
19724  {
19725  std::ostringstream error_message;
19726  error_message
19727  << "Have processed " << count_non_halo_el << " of "
19728  << target_domain_for_local_non_halo_element.size()
19729  << " target domains for local non-halo elelemts. \n "
19730  << "Very Odd -- we do (now) strip out the information for elements\n"
19731  << "that are removed in actions_before_distribute()...\n";
19732  throw OomphLibError(
19733  error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
19734  }
19735 #endif
19736 
19737  // Determine max. refinement level and origin/destination scheme
19738  // -------------------------------------------------------------
19739  // for all root/base elements
19740  // --------------------------
19741 
19742  // Allreduce to work out max max refinement level across all processors
19743  max_refinement_level_overall = 0;
19744 
19745  // Only perform this communications if necessary (it means if there
19746  // are structured meshes as part of the problem)
19747  if (are_there_structured_meshes)
19748  {
19749  MPI_Allreduce(&max_refinement_level,
19750  &max_refinement_level_overall,
19751  1,
19752  MPI_UNSIGNED,
19753  MPI_MAX,
19754  comm_pt->mpi_comm());
19755  } // if (are_there_structured_meshes)
19756 
19757  // Allreduce to tell everybody about the original and new domains
19758  // for root elements
19759  Vector<int> tmp_old_domain_for_base_element(n_base_element);
19760 
19761  // Only perform this communications if necessary (it means if there
19762  // are structured meshes as part of the problem)
19763  if (are_there_structured_meshes)
19764  {
19765  MPI_Allreduce(&old_domain_for_base_element_local[0],
19766  &tmp_old_domain_for_base_element[0],
19767  n_base_element,
19768  MPI_INT,
19769  MPI_MAX,
19770  comm_pt->mpi_comm());
19771  } // if (are_there_structured_meshes)
19772 
19773  Vector<int> tmp_new_domain_for_base_element(n_base_element);
19774  // Only perform this communications if necessary (it means if there
19775  // are structured meshes as part of the problem)
19776  if (are_there_structured_meshes)
19777  {
19778  MPI_Allreduce(&new_domain_for_base_element_local[0],
19779  &tmp_new_domain_for_base_element[0],
19780  n_base_element,
19781  MPI_INT,
19782  MPI_MAX,
19783  comm_pt->mpi_comm());
19784  } // if (are_there_structured_meshes)
19785 
19786  // Copy across (after optional sanity check)
19787  old_domain_for_base_element.resize(n_base_element);
19788  new_domain_for_base_element.resize(n_base_element);
19789  for (unsigned j = 0; j < n_base_element; j++)
19790  {
19791 #ifdef PARANOID
19792  if (tmp_old_domain_for_base_element[j] == -1)
19793  {
19794  std::ostringstream error_message;
19795  error_message << "Old domain for base element " << j << ": "
19796  << Base_mesh_element_pt[j]
19797  << "or its incarnation as refineable el: "
19798  << dynamic_cast<RefineableElement*>(
19800  << " which is of type "
19801  << typeid(*Base_mesh_element_pt[j]).name()
19802  << " does not\n"
19803  << "appear to have been assigned by any processor\n";
19804  throw OomphLibError(error_message.str(),
19805  OOMPH_CURRENT_FUNCTION,
19806  OOMPH_EXCEPTION_LOCATION);
19807  }
19808 #endif
19809  old_domain_for_base_element[j] = tmp_old_domain_for_base_element[j];
19810 #ifdef PARANOID
19811  if (tmp_new_domain_for_base_element[j] == -1)
19812  {
19813  std::ostringstream error_message;
19814  error_message << "New domain for base element " << j
19815  << "which is of type "
19816  << typeid(*Base_mesh_element_pt[j]).name()
19817  << " does not\n"
19818  << "appear to have been assigned by any processor\n";
19819  throw OomphLibError(error_message.str(),
19820  OOMPH_CURRENT_FUNCTION,
19821  OOMPH_EXCEPTION_LOCATION);
19822  }
19823 #endif
19824  new_domain_for_base_element[j] = tmp_new_domain_for_base_element[j];
19825  }
19826 
19827 
19828  // Loop over all processors and accumulate data to be sent
19829  //--------------------------------------------------------
19830  send_data.clear();
19831 
19832  // Only do each node once (per processor!)
19833  Vector<std::map<Node*, bool>> node_done(n_proc);
19834 
19835  // Loop over all processors. NOTE: We include current processor
19836  // since we have to refine local elements too -- store their data
19837  // in same data structure as the one used for off-processor elements.
19838  for (int rank = 0; rank < n_proc; rank++)
19839  {
19840  // Set the offset for the current processor
19841  send_displacement[rank] = send_data.size();
19842 
19843 #ifdef PARANOID
19844  // Check that total number of elements processed matches those
19845  // in individual batches
19846  unsigned total_nel = element_for_processor[rank].size();
19847 #endif
19848 
19849  // Counter for number of elements
19850  unsigned el_count = 0;
19851 
19852  // How many baches are there for current rank?
19853  unsigned nbatch = nelement_batch_for_processor[rank].size();
19854 
19855  // Add to vector of doubles to save on number of comms
19856  send_data.push_back(double(nbatch));
19857 
19858  // Loop over batches of elemnts associated with same root
19859  for (unsigned b = 0; b < nbatch; b++)
19860  {
19861  // How many elements are to be sent in this batch?
19862  unsigned nel = nelement_batch_for_processor[rank][b];
19863 
19864  // Get the unique number of the root element in unrefined mesh for
19865  // all the elements in this batch
19866  unsigned base_el_no =
19867  base_element_for_element_batch_for_processor[rank][b];
19868 
19869  // Add unsigneds to send data to minimise number of
19870  // communications
19871  send_data.push_back(double(nel));
19872  send_data.push_back(double(base_el_no));
19873 
19874  // Loop over batch of elements
19875  for (unsigned e = 0; e < nel; e++)
19876  {
19877  // Get element
19878  GeneralisedElement* el_pt = element_for_processor[rank][el_count];
19879 
19880  // FE?
19881  FiniteElement* fe_pt = dynamic_cast<FiniteElement*>(el_pt);
19882  if (fe_pt != 0)
19883  {
19884  // Loop over nodes
19885  unsigned nnod = fe_pt->nnode();
19886  for (unsigned j = 0; j < nnod; j++)
19887  {
19888  Node* nod_pt = fe_pt->node_pt(j);
19889 
19890  // Reconstruct the nodal values/position from the node's
19891  // possible hanging node representation to be on the safe side
19892  unsigned n_value = nod_pt->nvalue();
19893  unsigned nt = nod_pt->ntstorage();
19894  Vector<double> values(n_value);
19895  unsigned n_dim = nod_pt->ndim();
19896  Vector<double> position(n_dim);
19897 
19898  // Loop over all history values
19899  for (unsigned t = 0; t < nt; t++)
19900  {
19901  nod_pt->value(t, values);
19902  for (unsigned i = 0; i < n_value; i++)
19903  {
19904  nod_pt->set_value(t, i, values[i]);
19905  }
19906  nod_pt->position(t, position);
19907  for (unsigned i = 0; i < n_dim; i++)
19908  {
19909  nod_pt->x(t, i) = position[i];
19910  }
19911  }
19912 
19913 
19914  // Has the node already been done for current rank?
19915  if (!node_done[rank][nod_pt])
19916  {
19917  // Now it has been done
19918  node_done[rank][nod_pt] = true;
19919 
19920  // Store number of values (as double) to allow for resizing
19921  // before read (req'd in case we store data that
19922  // got introduced by attaching FaceElements to bulk)
19923  send_data.push_back(double(n_value));
19924 
19925  // Check if it's a boundary node
19926  BoundaryNodeBase* bnod_pt =
19927  dynamic_cast<BoundaryNodeBase*>(nod_pt);
19928 
19929  // Not a boundary node
19930  if (bnod_pt == 0)
19931  {
19932 #ifdef PARANOID
19933  // Record status for checking
19934  send_data.push_back(double(0));
19935 #endif
19936  }
19937  // Yes it's a boundary node
19938  else
19939  {
19940 #ifdef PARANOID
19941  // Record status for checking
19942  send_data.push_back(double(1));
19943 #endif
19944  // Get pointer to the map of indices associated with
19945  // additional values created by face elements
19946  std::map<unsigned, unsigned>* map_pt =
19948 
19949  // No additional values created
19950  if (map_pt == 0)
19951  {
19952  send_data.push_back(double(0));
19953  }
19954  // Created additional values
19955  else
19956  {
19957  // How many?
19958  send_data.push_back(double(map_pt->size()));
19959 
19960  // Loop over entries in map and add to send data
19961  for (std::map<unsigned, unsigned>::iterator p =
19962  map_pt->begin();
19963  p != map_pt->end();
19964  p++)
19965  {
19966  send_data.push_back(double((*p).first));
19967  send_data.push_back(double((*p).second));
19968  }
19969  }
19970  }
19971 
19972  // Add the actual values
19973  nod_pt->add_values_to_vector(send_data);
19974  }
19975  }
19976  }
19977 
19978  // Now add internal data
19979  el_pt->add_internal_data_values_to_vector(send_data);
19980 
19981  // Bump up counter in long vector of elements
19982  el_count++;
19983  }
19984  }
19985 
19986 
19987 #ifdef PARANOID
19988  // Check that total number of elements matches the total of those
19989  // in batches
19990  if (total_nel != el_count)
19991  {
19992  std::ostringstream error_message;
19993  error_message
19994  << "total_nel: " << total_nel << " "
19995  << " doesn't match total number of elements sent in batch: "
19996  << el_count << "\n";
19997  throw OomphLibError(error_message.str(),
19998  OOMPH_CURRENT_FUNCTION,
19999  OOMPH_EXCEPTION_LOCATION);
20000  }
20001 #endif
20002 
20003  // Find the number of data added to the vector
20004  send_n[rank] = send_data.size() - send_displacement[rank];
20005  }
20006  }
20007 
20008 
20009  //==========================================================================
20010  /// Get flat-packed refinement pattern for each root element in current
20011  /// mesh (labeled by unique number of root element in unrefined base mesh).
20012  /// The vector stored for each root element contains the following
20013  /// information:
20014  /// - First entry: Number of tree nodes (not just leaves!) in refinement
20015  /// tree emanating from this root [Zero if root element is not refineable]
20016  /// - Loop over all refinement levels
20017  /// - Loop over all tree nodes (not just leaves!)
20018  /// - If associated element exists when the mesh has been refined to
20019  /// this level (either because it has been refined to this level or
20020  /// because it's less refined): 1
20021  /// - If the element is to be refined: 1; else: 0
20022  /// - else (element doesn't exist when mesh is refined to this level
20023  /// (because it's more refined): 0
20024  /// .
20025  /// .
20026  /// .
20027  //==========================================================================
20029  const Vector<unsigned>& old_domain_for_base_element,
20030  const Vector<unsigned>& new_domain_for_base_element,
20031  const unsigned& max_refinement_level_overall,
20032  std::map<unsigned, Vector<unsigned>>& flat_packed_refinement_info_for_root)
20033  {
20034  // Map to store whether the root element has been visited yet
20035  std::map<RefineableElement*, bool> root_el_done;
20036 
20037  // Get the number of submeshs, if there are no submeshes, then
20038  // increase the counter so that the loop below also work for the only
20039  // one mesh in the problem
20040  unsigned n_mesh = nsub_mesh();
20041  if (n_mesh == 0)
20042  {
20043  n_mesh = 1;
20044  }
20045  // Go for the nonhalo elements only in the TreeBaseMeshes
20046  for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
20047  {
20048  // Only work with structured
20049  TriangleMeshBase* sub_mesh_pt =
20050  dynamic_cast<TriangleMeshBase*>(mesh_pt(i_mesh));
20051  if (!(sub_mesh_pt != 0))
20052  {
20053  const unsigned nele_submesh = mesh_pt(i_mesh)->nelement();
20054  for (unsigned e = 0; e < nele_submesh; e++)
20055  {
20056  // Get pointer to element
20057  GeneralisedElement* el_pt = mesh_pt(i_mesh)->element_pt(e);
20058 
20059  // Ignore halos
20060  if (!el_pt->is_halo())
20061  {
20062  // Is it refineable? No!
20063  RefineableElement* ref_el_pt =
20064  dynamic_cast<RefineableElement*>(el_pt);
20065  if (ref_el_pt == 0)
20066  {
20067  // The element is not refineable - stick a zero in refinement_info
20068  // indicating that there are no tree nodes following
20069  unsigned e = Base_mesh_element_number_plus_one[el_pt];
20070 #ifdef PARANOID
20071  if (e == 0)
20072  {
20073  throw OomphLibError("Base_mesh_element_number_plus_one[...]=0",
20074  OOMPH_CURRENT_FUNCTION,
20075  OOMPH_EXCEPTION_LOCATION);
20076  }
20077 #endif
20078  e -= 1;
20079  flat_packed_refinement_info_for_root[e].push_back(0);
20080  }
20081  // Refineable
20082  else
20083  {
20084  // Get the root element
20085  RefineableElement* root_el_pt = ref_el_pt->root_element_pt();
20086 
20087  // Has this root been visited yet?
20088  if (!root_el_done[root_el_pt])
20089  {
20090  // Get unique number of root element in base mesh
20091  unsigned root_element_number =
20093 
20094 #ifdef PARANOID
20095  if (root_element_number == 0)
20096  {
20097  throw OomphLibError(
20098  "Base_mesh_element_number_plus_one[...]=0",
20099  OOMPH_CURRENT_FUNCTION,
20100  OOMPH_EXCEPTION_LOCATION);
20101  }
20102 #endif
20103  root_element_number -= 1;
20104 
20105  // Get all the nodes associated with this root element
20106  Vector<Tree*> all_tree_nodes_pt;
20108  all_tree_nodes_pt);
20109 
20110  // How many tree nodes are there?
20111  unsigned n_tree_nodes = all_tree_nodes_pt.size();
20112  flat_packed_refinement_info_for_root[root_element_number]
20113  .push_back(n_tree_nodes);
20114 
20115  // Loop over all levels
20116  for (unsigned current_level = 0;
20117  current_level < max_refinement_level_overall;
20118  current_level++)
20119  {
20120  // Loop over all tree nodes
20121  for (unsigned e = 0; e < n_tree_nodes; e++)
20122  {
20123  // What's the level of this tree node?
20124  unsigned level = all_tree_nodes_pt[e]->level();
20125 
20126  // Element exists at this refinement level of the mesh
20127  // if it's at this level or it's at a lower level and a leaf
20128  if ((level == current_level) ||
20129  ((level < current_level) &&
20130  (all_tree_nodes_pt[e]->is_leaf())))
20131  {
20132  flat_packed_refinement_info_for_root[root_element_number]
20133  .push_back(1);
20134 
20135  // If it's at this level, and not a leaf, then it will
20136  // need to be refined in the new mesh
20137  if ((level == current_level) &&
20138  (!all_tree_nodes_pt[e]->is_leaf()))
20139  {
20140  flat_packed_refinement_info_for_root
20141  [root_element_number]
20142  .push_back(1);
20143  }
20144  // Element exists at this level and is a leaf so it
20145  // doesn't have to be refined
20146  else
20147  {
20148  flat_packed_refinement_info_for_root
20149  [root_element_number]
20150  .push_back(0);
20151  }
20152  }
20153  // Element does not exist at this level so it doesn't have
20154  // to be refined
20155  else
20156  {
20157  flat_packed_refinement_info_for_root[root_element_number]
20158  .push_back(0);
20159  }
20160  }
20161  }
20162  // Now we've done it
20163  root_el_done[root_el_pt] = true;
20164  }
20165  }
20166 
20167  } // if (!el_pt->is_halo())
20168  } // for (e < nele_submesh)
20169  } // if (!(sub_mesh_pt!=0))
20170  } // for (i_mesh < n_mesh)
20171  }
20172 
20173  //==========================================================================
20174  /// Load balance helper routine: Function performs max_level_overall
20175  /// successive refinements of the problem's mesh(es) using the following
20176  /// procdure: Given ID of root element, root_element_id, and current
20177  /// refinement level, level, the e-th entry in
20178  /// refinement_info_for_root_elements[root_element_id][level][e] is equal
20179  /// to 2 if the e-th element (using the enumeration when the mesh has been
20180  /// refined to the level-th level) is to be refined during the next
20181  /// refinement; it's 1 if it's not to be refined.
20182  //==========================================================================
20184  Vector<Vector<Vector<unsigned>>>& refinement_info_for_root_elements,
20185  const unsigned& max_level_overall)
20186  {
20187  // Loop over sub meshes
20188  unsigned n_sub_mesh = nsub_mesh();
20189  unsigned max_mesh = std::max(n_sub_mesh, unsigned(1));
20190  for (unsigned i_mesh = 0; i_mesh < max_mesh; i_mesh++)
20191  {
20192  // Choose the right mesh
20193  Mesh* my_mesh_pt = 0;
20194  if (n_sub_mesh == 0)
20195  {
20196  my_mesh_pt = mesh_pt();
20197  }
20198  else
20199  {
20200  my_mesh_pt = mesh_pt(i_mesh);
20201  }
20202 
20203  // Number of elements on this processor -- currently all elements
20204  // are "base" elements since the mesh hasn't been refined.
20205  unsigned n_el_on_this_proc = my_mesh_pt->nelement();
20206 
20207  // Storage for actual refinement pattern:
20208  // to_be_refined_on_this_proc[level][e] contains the element number
20209  // of the e-th element that is to refined at the level-th refinement level
20210  Vector<Vector<unsigned>> to_be_refined_on_this_proc(max_level_overall);
20211 
20212  // Count, at each level, the total number of elements in the mesh
20213  // (we can accumulate this because we know that elements are
20214  // enumerated tree by tree).
20215  Vector<unsigned> el_count_on_this_proc(max_level_overall, 0);
20216 
20217  // Loop over levels where refinement is taking place
20218  for (unsigned level = 0; level < max_level_overall; level++)
20219  {
20220  // Loop over roots = unrefined elements on this processor in order.
20221  // Note that this loops over the trees in unique order
20222  for (unsigned e = 0; e < n_el_on_this_proc; e++)
20223  {
20224  // Get the (root) element
20225  FiniteElement* el_pt = my_mesh_pt->finite_element_pt(e);
20226 
20227  // What is its unique number in the base mesh
20228  unsigned root_el_no = Base_mesh_element_number_plus_one[el_pt];
20229 #ifdef PARANOID
20230  if (root_el_no == 0)
20231  {
20232  throw OomphLibError("Base_mesh_element_number_plus_one[...]=0",
20233  OOMPH_CURRENT_FUNCTION,
20234  OOMPH_EXCEPTION_LOCATION);
20235  }
20236 #endif
20237  root_el_no -= 1;
20238 
20239  // Number of refinements to be performed starting from current
20240  // root element
20241  unsigned n_refinements =
20242  refinement_info_for_root_elements[root_el_no].size();
20243 
20244  // Perform refinement?
20245  if (level < n_refinements)
20246  {
20247  // Loop over elements at this level
20248  unsigned n_el =
20249  refinement_info_for_root_elements[root_el_no][level].size();
20250  for (unsigned ee = 0; ee < n_el; ee++)
20251  {
20252  // Refinement code 2: Element is to be refined at this
20253  // level
20254  if (refinement_info_for_root_elements[root_el_no][level][ee] == 2)
20255  {
20256  to_be_refined_on_this_proc[level].push_back(
20257  el_count_on_this_proc[level]);
20258  el_count_on_this_proc[level]++;
20259  }
20260  // Refinement code 1: Element should not be refined at this
20261  // level -- keep going
20262  else if (refinement_info_for_root_elements[root_el_no][level]
20263  [ee] == 1)
20264  {
20265  el_count_on_this_proc[level]++;
20266  }
20267  }
20268  }
20269 
20270  } // end of loop over elements on proc; all of which should be root
20271  }
20272 
20273  // Now do the actual refinement
20274  TreeBasedRefineableMeshBase* ref_mesh_pt =
20275  dynamic_cast<TreeBasedRefineableMeshBase*>(my_mesh_pt);
20276  if (ref_mesh_pt != 0)
20277  {
20278  ref_mesh_pt->refine_base_mesh(to_be_refined_on_this_proc);
20279  }
20280  }
20281 
20282  // Rebuild global mesh after refinement
20283  if (n_sub_mesh != 0)
20284  {
20285  // Rebuild the global mesh
20287  }
20288  }
20289 
20290 
20291  //====================================================================
20292  /// Helper function to re-setup the Base_mesh enumeration
20293  /// (used during load balancing) after pruning.
20294  //====================================================================
20296  {
20297  // Storage for number of processors and current processor
20298  int n_proc = this->communicator_pt()->nproc();
20299  int my_rank = this->communicator_pt()->my_rank();
20300 
20301  // Loop over sub meshes
20302  unsigned n_sub_mesh = nsub_mesh();
20303  unsigned max_mesh = std::max(n_sub_mesh, unsigned(1));
20304  for (unsigned i_mesh = 0; i_mesh < max_mesh; i_mesh++)
20305  {
20306  // Choose the right mesh
20307  Mesh* my_mesh_pt = 0;
20308  if (n_sub_mesh == 0)
20309  {
20310  my_mesh_pt = mesh_pt();
20311  }
20312  else
20313  {
20314  my_mesh_pt = mesh_pt(i_mesh);
20315  }
20316 
20317  // Only work with structured meshes
20318  TriangleMeshBase* sub_mesh_pt =
20319  dynamic_cast<TriangleMeshBase*>(my_mesh_pt);
20320  if (!(sub_mesh_pt != 0))
20321  {
20322  // Storage for number of data to be sent to each processor
20323  Vector<int> send_n(n_proc, 0);
20324 
20325  // Storage for all values to be sent to all processors
20326  Vector<unsigned> send_data;
20327 
20328  // Start location within send_data for data to be sent to each processor
20329  Vector<int> send_displacement(n_proc, 0);
20330 
20331  // Loop over all processors
20332  for (int rank = 0; rank < n_proc; rank++)
20333  {
20334  // Set the offset for the current processor
20335  send_displacement[rank] = send_data.size();
20336 
20337  // Don't bother to do anything if the processor in the loop is the
20338  // current processor
20339  if (rank != my_rank)
20340  {
20341  // Get root haloed elements with that processor
20342  Vector<GeneralisedElement*> root_haloed_elements_pt =
20343  my_mesh_pt->root_haloed_element_pt(rank);
20344  unsigned nel = root_haloed_elements_pt.size();
20345 
20346  // Store element numbers for send
20347  for (unsigned e = 0; e < nel; e++)
20348  {
20349  GeneralisedElement* el_pt = root_haloed_elements_pt[e];
20350  send_data.push_back(Base_mesh_element_number_plus_one[el_pt]);
20351  }
20352  }
20353 
20354  // Find the number of data added to the vector
20355  send_n[rank] = send_data.size() - send_displacement[rank];
20356  }
20357 
20358  // Storage for the number of data to be received from each processor
20359  Vector<int> receive_n(n_proc, 0);
20360 
20361  // Now send numbers of data to be sent between all processors
20362  MPI_Alltoall(&send_n[0],
20363  1,
20364  MPI_INT,
20365  &receive_n[0],
20366  1,
20367  MPI_INT,
20368  this->communicator_pt()->mpi_comm());
20369 
20370  // We now prepare the data to be received
20371  // by working out the displacements from the received data
20372  Vector<int> receive_displacement(n_proc, 0);
20373  int receive_data_count = 0;
20374  for (int rank = 0; rank < n_proc; ++rank)
20375  {
20376  // Displacement is number of data received so far
20377  receive_displacement[rank] = receive_data_count;
20378  receive_data_count += receive_n[rank];
20379  }
20380 
20381  // Now resize the receive buffer for all data from all processors
20382  // Make sure that it has a size of at least one
20383  if (receive_data_count == 0)
20384  {
20385  ++receive_data_count;
20386  }
20387  Vector<unsigned> receive_data(receive_data_count);
20388 
20389  // Make sure that the send buffer has size at least one
20390  // so that we don't get a segmentation fault
20391  if (send_data.size() == 0)
20392  {
20393  send_data.resize(1);
20394  }
20395 
20396  // Now send the data between all the processors
20397  MPI_Alltoallv(&send_data[0],
20398  &send_n[0],
20399  &send_displacement[0],
20400  MPI_UNSIGNED,
20401  &receive_data[0],
20402  &receive_n[0],
20403  &receive_displacement[0],
20404  MPI_UNSIGNED,
20405  this->communicator_pt()->mpi_comm());
20406 
20407  // Now use the received data to update the halo element numbers in
20408  // base mesh
20409  for (int send_rank = 0; send_rank < n_proc; send_rank++)
20410  {
20411  // Don't bother to do anything for the processor corresponding to the
20412  // current processor or if no data were received from this processor
20413  if ((send_rank != my_rank) && (receive_n[send_rank] != 0))
20414  {
20415  // Counter for the data within the large array
20416  unsigned count = receive_displacement[send_rank];
20417 
20418  // Get root halo elements with that processor
20419  Vector<GeneralisedElement*> root_halo_elements_pt =
20420  my_mesh_pt->root_halo_element_pt(send_rank);
20421  unsigned nel = root_halo_elements_pt.size();
20422 
20423  // Read in element numbers
20424  for (unsigned e = 0; e < nel; e++)
20425  {
20426  GeneralisedElement* el_pt = root_halo_elements_pt[e];
20427  unsigned el_number_plus_one = receive_data[count++];
20428  Base_mesh_element_number_plus_one[el_pt] = el_number_plus_one;
20429  Base_mesh_element_pt[el_number_plus_one - 1] = el_pt;
20430  }
20431  }
20432 
20433  } // End of data is received
20434 
20435  } // if (!(sub_mesh_pt!=0))
20436 
20437  } // for (i_mesh<max_mesh)
20438  }
20439 
20440 #endif
20441 
20442  /// Instantiation of public flag to allow suppression of warning
20443  /// messages re reading in unstructured meshes during restart.
20445  false;
20446 
20447 
20448 } // namespace oomph
e
Definition: cfortran.h:571
cstr elem_len * i
Definition: cfortran.h:603
char t
Definition: cfortran.h:568
Function-type-object to perform absolute comparison of objects. Apparently this inlines better.
A class that is used to define the functions used to assemble the elemental contributions to the resi...
virtual unsigned ndof(GeneralisedElement *const &elem_pt)
Return the number of degrees of freedom in the element elem_pt.
virtual void synchronise()
Function that is used to perform any synchronisation required during the solution.
virtual int bifurcation_type() const
Return an unsigned integer to indicate whether the handler is a bifurcation tracking handler....
virtual void get_hessian_vector_products(GeneralisedElement *const &elem_pt, Vector< double > const &Y, DenseMatrix< double > const &C, DenseMatrix< double > &product)
Calculate the product of the Hessian (derivative of Jacobian with respect to all variables) an eigenv...
virtual double * bifurcation_parameter_pt() const
Return a pointer to the bifurcation parameter in bifurcation tracking problems.
virtual void get_all_vectors_and_matrices(GeneralisedElement *const &elem_pt, Vector< Vector< double >> &vec, Vector< DenseMatrix< double >> &matrix)
Calculate all desired vectors and matrices provided by the element elem_pt.
virtual void get_eigenfunction(Vector< DoubleVector > &eigenfunction)
Return the eigenfunction(s) associated with the bifurcation that has been detected in bifurcation tra...
virtual void get_residuals(GeneralisedElement *const &elem_pt, Vector< double > &residuals)
Return the contribution to the residuals of the element elem_pt.
virtual unsigned long eqn_number(GeneralisedElement *const &elem_pt, const unsigned &ieqn_local)
Return the global equation number of the local unknown ieqn_local in elem_pt.
virtual void get_jacobian(GeneralisedElement *const &elem_pt, Vector< double > &residuals, DenseMatrix< double > &jacobian)
Calculate the elemental Jacobian matrix "d equation / d variable" for elem_pt.
A custom linear solver class that is used to solve a block-factorised version of the Hopf bifurcation...
A class that contains the information required by Nodes that are located on Mesh boundaries....
Definition: nodes.h:1996
std::map< unsigned, unsigned > *& index_of_first_value_assigned_by_face_element_pt()
Return pointer to the map giving the index of the first face element value.
Definition: nodes.h:2046
//////////////////////////////////////////////////////////////// ////////////////////////////////////...
Definition: matrices.h:2791
void build_without_copy(T *value, int *row_index, int *column_start, const unsigned long &nnz, const unsigned long &n, const unsigned long &m)
Function to build matrix from pointers to arrays which hold the column starts, row indices and non-ze...
Definition: matrices.h:3199
A class for compressed row matrices. This is a distributable object.
Definition: matrices.h:888
void redistribute(const LinearAlgebraDistribution *const &dist_pt)
The contents of the matrix are redistributed to match the new distribution. In a non-MPI build this m...
Definition: matrices.cc:2575
void build_without_copy(const unsigned &ncol, const unsigned &nnz, double *value, int *column_index, int *row_start)
keeps the existing distribution and just matrix that is stored without copying the matrix data
Definition: matrices.cc:1710
unsigned long nrow() const
Return the number of rows of the matrix.
Definition: matrices.h:1002
void build(const LinearAlgebraDistribution *distribution_pt, const unsigned &ncol, const Vector< double > &value, const Vector< int > &column_index, const Vector< int > &row_start)
build method: vector of values, vector of column indices, vector of row starts and number of rows and...
Definition: matrices.cc:1672
void set_consistent_pinned_values(Data *const &data_pt)
Set consistent values of the derivatives and current value when the data is pinned....
A Base class for DGElements.
Definition: dg_elements.h:161
void disable_mass_matrix_reuse()
Function that disables the reuse of the mass matrix.
Definition: dg_elements.h:240
void enable_mass_matrix_reuse()
Function that allows the reuse of the mass matrix.
Definition: dg_elements.h:233
virtual void get_inverse_mass_matrix_times_residuals(Vector< double > &minv_res)
Function that returns the current value of the residuals multiplied by the inverse mass matrix (virtu...
Definition: dg_elements.cc:614
A class that represents a collection of data; each Data object may contain many different individual ...
Definition: nodes.h:86
long & eqn_number(const unsigned &i)
Return the equation number of the i-th stored variable.
Definition: nodes.h:367
virtual void add_eqn_numbers_to_vector(Vector< long > &vector_of_eqn_numbers)
Add all equation numbers to the vector in the internal storage order.
Definition: nodes.cc:1236
double * value_pt(const unsigned &i) const
Return the pointer to the i-the stored value. Typically this is required when direct access to the st...
Definition: nodes.h:324
void copy(Data *orig_data_pt)
Copy Data values from specified Data object.
Definition: nodes.cc:601
void set_value(const unsigned &i, const double &value_)
Set the i-th stored data value to specified value. The only reason that we require an explicit set fu...
Definition: nodes.h:271
unsigned nvalue() const
Return number of values stored in data object (incl pinned ones).
Definition: nodes.h:483
double value(const unsigned &i) const
Return i-th stored value. This function is not virtual so that it can be inlined. This means that if ...
Definition: nodes.h:293
unsigned ntstorage() const
Return total number of doubles stored per value to record time history of each value (one for steady ...
Definition: nodes.cc:879
virtual void read_eqn_numbers_from_vector(const Vector< long > &vector_of_eqn_numbers, unsigned &index)
Read all equation numbers from the vector starting from index. On return the index will be set to the...
Definition: nodes.cc:1274
bool is_pinned(const unsigned &i) const
Test whether the i-th variable is pinned (1: true; 0: false).
Definition: nodes.h:417
Class of matrices containing doubles, and stored as a DenseMatrix<double>, but with solving functiona...
Definition: matrices.h:1271
void initialise(const T &val)
Initialize all values in the matrix to val.
Definition: matrices.h:514
void resize(const unsigned long &n)
Resize to a square nxn matrix; any values already present will be transfered.
Definition: matrices.h:498
LinearAlgebraDistribution * distribution_pt() const
access to the LinearAlgebraDistribution
unsigned nrow() const
access function to the number of global rows.
bool distribution_built() const
if the communicator_pt is null then the distribution is not setup then false is returned,...
unsigned nrow_local() const
access function for the num of local rows on this processor.
Information for documentation of results: Directory and file number to enable output in the form RESL...
bool is_doc_enabled() const
Are we documenting?
void disable_doc()
Disable documentation.
std::string & label()
String used (e.g.) for labeling output files.
std::string directory() const
Output directory.
unsigned & number()
Number used (e.g.) for labeling output files.
A class that stores the halo/haloed entries required when using a DoubleVectorWithHaloEntries....
void setup_halo_dofs(const std::map< unsigned, double * > &halo_data_pt, Vector< double * > &halo_dof_pt)
Function that sets up a vector of pointers to halo data, index using the scheme in Local_index.
===================================================================== An extension of DoubleVector th...
void build_halo_scheme(DoubleVectorHaloScheme *const &halo_scheme_pt)
Construct the halo scheme and storage for the halo data.
void sum_all_halo_and_haloed_values()
Sum all the data, store in the master (haloed) data and then synchronise.
double & global_value(const unsigned &i)
Direct access to global entry.
A vector in the mathematical sense, initially developed for linear algebra type applications....
Definition: double_vector.h:58
void initialise(const double &v)
initialise the whole vector with value v
void set_external_values(const LinearAlgebraDistribution *const &dist_pt, double *external_values, bool delete_external_values)
Allows are external data to be used by this vector. WARNING: The size of the external data must corre...
double max() const
returns the maximum coefficient
void build(const DoubleVector &old_vector)
Just copys the argument DoubleVector.
double * values_pt()
access function to the underlying values
void redistribute(const LinearAlgebraDistribution *const &dist_pt)
The contents of the vector are redistributed to match the new distribution. In a non-MPI rebuild this...
bool built() const
double dot(const DoubleVector &vec) const
compute the dot product of this vector with the vector vec.
void clear()
wipes the DoubleVector
A class that is used to define the functions used to assemble the elemental contributions to the mass...
virtual void solve_eigenproblem(Problem *const &problem_pt, const int &n_eval, Vector< std::complex< double >> &eigenvalue, Vector< DoubleVector > &eigenvector_real, Vector< DoubleVector > &eigenvector_imag, const bool &do_adjoint_problem=false)
Solve the real eigenproblem that is assembled by elements in a mesh in a Problem object....
Definition: eigen_solver.h:106
virtual void solve_eigenproblem_legacy(Problem *const &problem_pt, const int &n_eval, Vector< std::complex< double >> &eigenvalue, Vector< DoubleVector > &eigenvector, const bool &do_adjoint_problem=false)=0
Eigensolver. This takes a pointer to a problem and returns a vector of complex numbers representing t...
Base class for spatial error estimators.
void get_element_errors(Mesh *&mesh_pt, Vector< double > &elemental_error)
Compute the elemental error-measures for a given mesh and store them in a vector.
A class that is used to define the functions used to assemble and invert the mass matrix when taking ...
A Base class for explicit timesteppers.
virtual void timestep(ExplicitTimeSteppableObject *const &object_pt, const double &dt)=0
Pure virtual function that is used to advance time in the object.
FaceElements are elements that coincide with the faces of higher-dimensional "bulk" elements....
Definition: elements.h:4338
int & face_index()
Index of the face (a number that uniquely identifies the face in the element)
Definition: elements.h:4626
FiniteElement *& bulk_element_pt()
Pointer to higher-dimensional "bulk" element.
Definition: elements.h:4735
A general Finite Element class.
Definition: elements.h:1313
Node *& node_pt(const unsigned &n)
Return a pointer to the local node n.
Definition: elements.h:2175
virtual void output(std::ostream &outfile)
Output the element data — typically the values at the nodes in a format suitable for post-processing.
Definition: elements.h:3050
unsigned nnode() const
Return the number of nodes.
Definition: elements.h:2210
A Generalised Element class.
Definition: elements.h:73
bool is_halo() const
Is this element a halo?
Definition: elements.h:1163
void read_internal_eqn_numbers_from_vector(const Vector< long > &vector_of_eqn_numbers, unsigned &index)
Read all equation numbers associated with internal data from the vector starting from index....
Definition: elements.cc:670
unsigned ndof() const
Return the number of equations/dofs in the element.
Definition: elements.h:835
unsigned long eqn_number(const unsigned &ieqn_local) const
Return the global equation number corresponding to the ieqn_local-th local equation number.
Definition: elements.h:704
void add_internal_data_values_to_vector(Vector< double > &vector_of_values)
Add all internal data and time history values to the vector in the internal storage order.
Definition: elements.cc:628
Data *& internal_data_pt(const unsigned &i)
Return a pointer to i-th internal data object.
Definition: elements.h:622
unsigned ninternal_data() const
Return the number of internal data objects.
Definition: elements.h:823
void add_internal_eqn_numbers_to_vector(Vector< long > &vector_of_eqn_numbers)
Add all equation numbers associated with internal data to the vector in the internal storage order.
Definition: elements.cc:655
virtual void complete_setup_of_dependencies()
Complete the setup of any additional dependencies that the element may have. Empty virtual function t...
Definition: elements.h:974
void read_internal_data_values_from_vector(const Vector< double > &vector_of_values, unsigned &index)
Read all internal data and time history values from the vector starting from index....
Definition: elements.cc:642
Class that contains data for hanging nodes.
Definition: nodes.h:742
double const & master_weight(const unsigned &i) const
Return weight for dofs on i-th master node.
Definition: nodes.h:808
Node *const & master_node_pt(const unsigned &i) const
Return a pointer to the i-th master node.
Definition: nodes.h:791
unsigned nmaster() const
Return the number of master nodes.
Definition: nodes.h:785
void set_master_node_pt(const unsigned &i, Node *const &master_node_pt, const double &weight)
Set the pointer to the i-th master node and its weight.
Definition: nodes.cc:1474
A class that is used to assemble the augmented system that defines a Hopf bifurcation....
Class for the LAPACK QZ eigensolver.
Definition: eigen_solver.h:318
Describes the distribution of a distributable linear algebra type object. Typically this is a contain...
bool distributed() const
access function to the distributed - indicates whether the distribution is serial or distributed
unsigned first_row() const
access function for the first row on this processor. If not distributed then this is just zero.
OomphCommunicator * communicator_pt() const
const access to the communicator pointer
void build(const OomphCommunicator *const comm_pt, const unsigned &first_row, const unsigned &nrow_local, const unsigned &nrow=0)
Sets the distribution. Takes first_row, nrow_local and nrow as arguments. If nrow is not provided or ...
unsigned nrow() const
access function to the number of global rows.
unsigned nrow_local() const
access function for the num of local rows on this processor. If no MPI then Nrow is returned.
unsigned rank_of_global_row(const unsigned i) const
return the processor rank of the global row number i
virtual void solve(Problem *const &problem_pt, DoubleVector &result)=0
Solver: Takes pointer to problem and returns the results vector which contains the solution of the li...
virtual void enable_resolve()
Enable resolve (i.e. store matrix and/or LU decomposition, say) Virtual so it can be overloaded to pe...
virtual void enable_computation_of_gradient()
function to enable the computation of the gradient required for the globally convergent Newton method
virtual void resolve(const DoubleVector &rhs, DoubleVector &result)
Resolve the system defined by the last assembled jacobian and the rhs vector. Solution is returned in...
void get_gradient(DoubleVector &gradient)
function to access the gradient, provided it has been computed
void reset_gradient()
function to reset the size of the gradient before each Newton solve
virtual void disable_resolve()
Disable resolve (i.e. store matrix and/or LU decomposition, say) This function simply resets an inter...
bool is_resolve_enabled() const
Boolean flag indicating if resolves are enabled.
static bool mpi_has_been_initialised()
return true if MPI has been initialised
static OomphCommunicator * communicator_pt()
access to global communicator. This is the oomph-lib equivalent of MPI_COMM_WORLD
A general mesh class.
Definition: mesh.h:67
void set_external_halo_node_pt(const unsigned &p, const Vector< Node * > &external_halo_node_pt)
Set vector of external halo node in this Mesh whose non-halo external counterpart is held on processo...
Definition: mesh.h:2398
Node * halo_node_pt(const unsigned &p, const unsigned &j)
Access fct to the j-th halo node in this Mesh whose non-halo counterpart is held on processor p.
Definition: mesh.h:1914
bool does_pointer_correspond_to_mesh_data(double *const &parameter_pt)
Does the double pointer correspond to any mesh data.
Definition: mesh.cc:2471
void remove_boundary_node(const unsigned &b, Node *const &node_pt)
Remove a node from the boundary b.
Definition: mesh.cc:221
bool is_mesh_distributed() const
Boolean to indicate if Mesh has been distributed.
Definition: mesh.h:1588
unsigned nexternal_haloed_element()
Total number of external haloed elements in this Mesh.
Definition: mesh.h:2267
Node * haloed_node_pt(const unsigned &p, const unsigned &j)
Access fct to the j-th haloed node in this Mesh whose halo counterpart is held on processor p.
Definition: mesh.h:2014
unsigned nexternal_halo_node()
Total number of external halo nodes in this Mesh.
Definition: mesh.h:2308
void flush_element_and_node_storage()
Flush storage for elements and nodes by emptying the vectors that store the pointers to them....
Definition: mesh.h:407
Node *& external_halo_node_pt(const unsigned &p, const unsigned &j)
Access fct to the j-th external halo node in this Mesh whose non-halo external counterpart is held on...
Definition: mesh.h:2377
FiniteElement * finite_element_pt(const unsigned &e) const
Upcast (downcast?) to FiniteElement (needed to access FiniteElement member functions).
Definition: mesh.h:473
void check_halo_schemes(DocInfo &doc_info, double &max_permitted_error_for_halo_check)
Check halo and shared schemes on the mesh.
Definition: mesh.cc:6881
virtual void set_mesh_level_time_stepper(TimeStepper *const &time_stepper_pt, const bool &preserve_existing_data)
Function that can be used to set any additional timestepper data stored at the Mesh (as opposed to no...
Definition: mesh.cc:2402
Vector< GeneralisedElement * > haloed_element_pt(const unsigned &p)
Return vector of haloed elements in this Mesh whose haloing counterpart is held on processor p.
Definition: mesh.h:1779
void describe_local_dofs(std::ostream &out, const std::string &current_string) const
Function to describe the local dofs of the elements. The ostream specifies the output stream to which...
Definition: mesh.cc:746
GeneralisedElement *& external_halo_element_pt(const unsigned &p, const unsigned &e)
Access fct to the e-th external halo element in this Mesh whose non-halo counterpart is held on proce...
Definition: mesh.h:2251
void set_nodal_and_elemental_time_stepper(TimeStepper *const &time_stepper_pt, const bool &preserve_existing_data)
Set the timestepper associated with all nodal and elemental data stored in the mesh.
Definition: mesh.h:1032
void set_external_haloed_node_pt(const unsigned &p, const Vector< Node * > &external_haloed_node_pt)
Set vector of external haloed node in this Mesh whose halo external counterpart is held on processor ...
Definition: mesh.h:2466
Node *& external_haloed_node_pt(const unsigned &p, const unsigned &j)
Access fct to the j-th external haloed node in this Mesh whose halo external counterpart is held on p...
Definition: mesh.h:2441
void shift_time_values()
Shift time-dependent data along for next timestep: Deal with nodal Data/positions and the element's i...
Definition: mesh.cc:2326
void prune_halo_elements_and_nodes(Vector< GeneralisedElement * > &deleted_element_pt, const bool &report_stats=false)
(Irreversibly) prune halo(ed) elements and nodes, usually after another round of refinement,...
Definition: mesh.h:1667
void calculate_predictions()
Calculate predictions for all Data and positions associated with the mesh, usually used in adaptive t...
Definition: mesh.cc:2366
void describe_dofs(std::ostream &out, const std::string &current_string) const
Function to describe the dofs of the Mesh. The ostream specifies the output stream to which the descr...
Definition: mesh.cc:711
GeneralisedElement *& element_pt(const unsigned long &e)
Return pointer to element e.
Definition: mesh.h:448
unsigned long nnode() const
Return number of nodes in the mesh.
Definition: mesh.h:596
void delete_all_external_storage()
Wipe the storage for all externally-based elements.
Definition: mesh.cc:9190
unsigned nnon_halo_element()
Total number of non-halo elements in this mesh (Costly call computes result on the fly)
Definition: mesh.h:1817
void get_all_halo_data(std::map< unsigned, double * > &map_of_halo_data)
Get all the halo data stored in the mesh and add pointers to the data to the map, indexed by global e...
Definition: mesh.cc:4749
unsigned nhalo_node()
Total number of halo nodes in this Mesh.
Definition: mesh.h:1878
void set_communicator_pt(OomphCommunicator *comm_pt)
Function to set communicator (mesh is assumed to be distributed if the communicator pointer is non-nu...
Definition: mesh.h:1615
void output(std::ostream &outfile)
Output for all elements.
Definition: mesh.cc:2027
void assign_initial_values_impulsive()
Assign initial values for an impulsive start.
Definition: mesh.cc:2288
Node *& node_pt(const unsigned long &n)
Return pointer to global node n.
Definition: mesh.h:436
void assign_local_eqn_numbers(const bool &store_local_dof_pt)
Assign the local equation numbers in all elements If the boolean argument is true then also store poi...
Definition: mesh.cc:765
virtual void read(std::ifstream &restart_file)
Read solution from restart file.
Definition: mesh.cc:1130
void set_consistent_pinned_values_for_continuation(ContinuationStorageScheme *const &continuation_stepper_pt)
Set consistent values for pinned data in continuation.
Definition: mesh.cc:2436
unsigned nhaloed_node()
Total number of haloed nodes in this Mesh.
Definition: mesh.h:1985
unsigned nexternal_haloed_node()
Total number of external haloed nodes in this Mesh.
Definition: mesh.h:2412
unsigned long assign_global_eqn_numbers(Vector< double * > &Dof_pt)
Assign the global equation numbers in the Data stored at the nodes and also internal element Data....
Definition: mesh.cc:677
virtual void distribute(OomphCommunicator *comm_pt, const Vector< unsigned > &element_domain, Vector< GeneralisedElement * > &deleted_element_pt, DocInfo &doc_info, const bool &report_stats, const bool &overrule_keep_as_halo_element_status)
Distribute the problem and doc; make this virtual to allow overloading for particular meshes where fu...
Definition: mesh.cc:4959
void null_external_halo_node(const unsigned &p, Node *nod_pt)
Null out specified external halo node (used when deleting duplicates)
Definition: mesh.cc:8569
Vector< GeneralisedElement * > root_haloed_element_pt(const unsigned &p)
Vector of pointers to root haloed elements in this Mesh whose non-halo counterpart is held on process...
Definition: mesh.h:1951
GeneralisedElement *& external_haloed_element_pt(const unsigned &p, const unsigned &e)
Access fct to the e-th external haloed element in this Mesh whose non-halo counterpart is held on pro...
Definition: mesh.h:2296
unsigned long nelement() const
Return number of elements in the mesh.
Definition: mesh.h:590
Vector< GeneralisedElement * > root_halo_element_pt(const unsigned &p)
Vector of pointers to root halo elements in this Mesh whose non-halo counterpart is held on processor...
Definition: mesh.h:1854
void merge_meshes(const Vector< Mesh * > &sub_mesh_pt)
Merge meshes. Note: This simply merges the meshes' elements and nodes (ignoring duplicates; no bounda...
Definition: mesh.cc:65
unsigned nexternal_halo_element()
Total number of external halo elements in this Mesh.
Definition: mesh.h:2222
virtual void dump(std::ofstream &dump_file, const bool &use_old_ordering=true) const
Dump the data in the mesh into a file for restart.
Definition: mesh.cc:1088
Vector< GeneralisedElement * > halo_element_pt(const unsigned &p)
Return vector of halo elements in this Mesh whose non-halo counterpart is held on processor p.
Definition: mesh.h:1740
A class to handle errors in the Newton solver.
Definition: problem.h:3081
bool linear_solver_error
Error in the linear solver.
Definition: problem.h:3084
unsigned iterations
Max. # of iterations performed when the Newton solver died.
Definition: problem.h:3087
double maxres
Max. residual when Newton solver died.
Definition: problem.h:3090
Nodes are derived from Data, but, in addition, have a definite (Eulerian) position in a space of a gi...
Definition: nodes.h:906
double & x(const unsigned &i)
Return the i-th nodal coordinate.
Definition: nodes.h:1060
double & x_gen(const unsigned &k, const unsigned &i)
Reference to the generalised position x(k,i). ‘Type’: k; Coordinate direction: i.
Definition: nodes.h:1126
HangInfo *const & hanging_pt() const
Return pointer to hanging node data (this refers to the geometric hanging node status) (const version...
Definition: nodes.h:1228
void copy(Node *orig_node_pt)
Copy all nodal data from specified Node object.
Definition: nodes.cc:1916
void add_values_to_vector(Vector< double > &vector_of_values)
Add all data and time history values to the vector. Overloaded to add the position information as wel...
Definition: nodes.cc:2777
virtual void get_boundaries_pt(std::set< unsigned > *&boundaries_pt)
Return a pointer to set of mesh boundaries that this node occupies; this will be overloaded by Bounda...
Definition: nodes.h:1365
void position(Vector< double > &pos) const
Compute Vector of nodal positions either directly or via hanging node representation.
Definition: nodes.cc:2499
unsigned ndim() const
Return (Eulerian) spatial dimension of the node.
Definition: nodes.h:1054
void read_values_from_vector(const Vector< double > &vector_of_values, unsigned &index)
Read all data and time history values from the vector starting from index. On return the index will b...
Definition: nodes.cc:2832
void resize(const unsigned &n_value)
Resize the number of equations.
Definition: nodes.cc:2167
unsigned nposition_type() const
Number of coordinate types needed in the mapping between local and global coordinates.
Definition: nodes.h:1016
bool is_hanging() const
Test whether the node is geometrically hanging.
Definition: nodes.h:1285
double value(const unsigned &i) const
Return i-th value (dofs or pinned) at this node either directly or via hanging node representation....
Definition: nodes.cc:2408
An oomph-lib wrapper to the MPI_Comm communicator object. Just contains an MPI_Comm object (which is ...
Definition: communicator.h:54
std::ostream *& stream_pt()
Access function for the stream pointer.
An OomphLibError object which should be thrown when an run-time error is encountered....
An OomphLibWarning object which should be created as a temporary object to issue a warning....
A class that is used to assemble the residuals in parallel by overloading the get_all_vectors_and_mat...
A class that is used to define the functions used when assembling the derivatives of the residuals wi...
////////////////////////////////////////////////////////////////// //////////////////////////////////...
Definition: problem.h:151
virtual void actions_after_implicit_timestep()
Actions that should be performed after each implicit time step. This is needed when one wants to solv...
Definition: problem.h:1070
bool Always_take_one_newton_step
Boolean to indicate whether a Newton step should be taken even if the initial residuals are below the...
Definition: problem.h:2326
bool Jacobian_reuse_is_enabled
Is re-use of Jacobian in Newton iteration enabled? Default: false.
Definition: problem.h:618
double *& dof_pt(const unsigned &i)
Pointer to i-th dof in the problem.
Definition: problem.h:1829
virtual void actions_after_newton_solve()
Any actions that are to be performed after a complete Newton solve, e.g. post processing....
Definition: problem.h:1038
void parallel_sparse_assemble(const LinearAlgebraDistribution *const &dist_pt, Vector< int * > &column_or_row_index, Vector< int * > &row_or_column_start, Vector< double * > &value, Vector< unsigned > &nnz, Vector< double * > &residuals)
Helper method to assemble CRDoubleMatrices from distributed on multiple processors.
Definition: problem.cc:6533
void describe_dofs(std::ostream &out= *(oomph_info.stream_pt())) const
Function to describe the dofs in terms of the global equation number, i.e. what type of value (nodal ...
Definition: problem.cc:2445
virtual void sparse_assemble_row_or_column_compressed(Vector< int * > &column_or_row_index, Vector< int * > &row_or_column_start, Vector< double * > &value, Vector< unsigned > &nnz, Vector< double * > &residual, bool compressed_row_flag)
Protected helper function that is used to assemble the Jacobian matrix in the case when the storage i...
Definition: problem.cc:4461
bool Store_local_dof_pt_in_elements
Boolean to indicate whether local dof pointers should be stored in the elements.
Definition: problem.h:226
virtual void actions_before_newton_step()
Any actions that are to be performed before each individual Newton step. Most likely to be used for d...
Definition: problem.h:1053
void remove_duplicate_data(Mesh *const &mesh_pt, bool &actually_removed_some_data)
Private helper function to remove repeated data in external haloed elements in specified mesh....
Definition: problem.cc:2654
bool Bifurcation_detection
Boolean to control bifurcation detection via determinant of Jacobian.
Definition: problem.h:790
void solve_adjoint_eigenproblem_legacy(const unsigned &n_eval, Vector< std::complex< double >> &eigenvalue, Vector< DoubleVector > &eigenvector, const bool &make_timesteppers_steady=true)
Solve an adjoint eigenvalue problem using the same procedure as solve_eigenproblem....
Definition: problem.cc:8479
void adapt()
Adapt problem: Perform mesh adaptation for (all) refineable (sub)mesh(es), based on their own error e...
Definition: problem.h:2988
virtual void actions_before_newton_solve()
Any actions that are to be performed before a complete Newton solve (e.g. adjust boundary conditions)...
Definition: problem.h:1032
Vector< unsigned > First_el_for_assembly
First element to be assembled by given processor for non-distributed problem (only kept up to date wh...
Definition: problem.h:519
unsigned long assign_eqn_numbers(const bool &assign_local_eqn_numbers=true)
Assign all equation numbers for problem: Deals with global data (= data that isn't attached to any el...
Definition: problem.cc:2075
void refine_uniformly_aux(const Vector< unsigned > &nrefine_for_mesh, DocInfo &doc_info, const bool &prune)
Helper function to do compund refinement of (all) refineable (sub)mesh(es) uniformly as many times as...
Definition: problem.cc:15638
void build_global_mesh()
Build the global mesh by combining the all the submeshes. Note: The nodes boundary information refers...
Definition: problem.cc:1579
unsigned unrefine_uniformly()
Refine (all) refineable (sub)mesh(es) uniformly and rebuild problem. Return 0 for success,...
Definition: problem.cc:16024
void assign_initial_values_impulsive()
Initialise data and nodal positions to simulate impulsive start from initial configuration/solution.
Definition: problem.cc:11692
double & dof_current(const unsigned &i)
Access function to the current value of the i-th (local) dof at the start of a continuation step.
Definition: problem.h:1182
double Theta_squared
Value of the scaling parameter required so that the parameter occupies the desired proportion of the ...
Definition: problem.h:742
LinearAlgebraDistribution *const & dof_distribution_pt() const
Return the pointer to the dof distribution (read-only)
Definition: problem.h:1672
virtual void get_eigenproblem_matrices(CRDoubleMatrix &mass_matrix, CRDoubleMatrix &main_matrix, const double &shift=0.0)
Get the matrices required by a eigensolver. If the shift parameter is non-zero the second matrix will...
Definition: problem.cc:8602
Vector< double > Dof_derivative
Storage for the derivative of the problem variables wrt arc-length.
Definition: problem.h:774
virtual void sparse_assemble_row_or_column_compressed_with_lists(Vector< int * > &column_or_row_index, Vector< int * > &row_or_column_start, Vector< double * > &value, Vector< unsigned > &nnz, Vector< double * > &residual, bool compressed_row_flag)
Private helper function that is used to assemble the Jacobian matrix in the case when the storage is ...
Definition: problem.cc:4907
virtual void sparse_assemble_row_or_column_compressed_with_two_arrays(Vector< int * > &column_or_row_index, Vector< int * > &row_or_column_start, Vector< double * > &value, Vector< unsigned > &nnz, Vector< double * > &residual, bool compressed_row_flag)
Private helper function that is used to assemble the Jacobian matrix in the case when the storage is ...
Definition: problem.cc:6038
friend class BlockHopfLinearSolver
Definition: problem.h:162
void synchronise_dofs(const bool &do_halos, const bool &do_external_halos)
Synchronise the degrees of freedom by overwriting the haloed values with their non-halo counterparts ...
Definition: problem.cc:16655
virtual void actions_before_distribute()
Actions to be performed before a (mesh) distribution.
Definition: problem.h:1116
friend class FoldHandler
Definition: problem.h:153
Distributed_problem_matrix_distribution Dist_problem_matrix_distribution
The distributed matrix distribution method 1 - Automatic - the Problem distribution is employed,...
Definition: problem.h:956
DoubleVectorHaloScheme * Halo_scheme_pt
Pointer to the halo scheme for any global vectors that have the Dof_distribution.
Definition: problem.h:577
virtual void actions_after_change_in_global_parameter(double *const &parameter_pt)
Actions that are to be performed when the global parameter addressed by parameter_pt has been changed...
Definition: problem.h:1133
void p_refine_selected_elements(const Vector< unsigned > &elements_to_be_refined)
p-refine (one and only!) mesh by refining the elements identified by their numbers relative to the pr...
Definition: problem.cc:15352
void setup_dof_halo_scheme()
Function that is used to setup the halo scheme.
Definition: problem.cc:386
unsigned long ndof() const
Return the number of dofs.
Definition: problem.h:1678
void p_unrefine_uniformly(DocInfo &doc_info)
p-unrefine (all) p-refineable (sub)mesh(es) uniformly and rebuild problem.
Definition: problem.cc:16153
virtual void build_mesh()
Function to build the Problem's base mesh; this must be supplied by the user if they wish to use the ...
Definition: problem.h:1365
static bool Suppress_warning_about_actions_before_read_unstructured_meshes
Flag to allow suppression of warning messages re reading in unstructured meshes during restart.
Definition: problem.h:317
void solve_adjoint_eigenproblem(const unsigned &n_eval, Vector< std::complex< double >> &eigenvalue, Vector< DoubleVector > &eigenvector_real, Vector< DoubleVector > &eigenvector_imag, const bool &steady=true)
Solve an adjoint eigenvalue problem using the same procedure as solve_eigenproblem....
Definition: problem.cc:8536
bool Use_default_partition_in_load_balance
Flag to use "default partition" during load balance. Should only be set to true when run in validatio...
Definition: problem.h:514
void bifurcation_adapt_helper(unsigned &n_refined, unsigned &n_unrefined, const unsigned &bifurcation_type, const bool &actually_adapt=true)
A function that is used to adapt a bifurcation-tracking problem, which requires separate interpolatio...
Definition: problem.cc:13547
friend class AugmentedBlockFoldLinearSolver
Definition: problem.h:160
Vector< double > Elemental_assembly_time
Storage for assembly times (used for load balancing)
Definition: problem.h:573
bool Jacobian_has_been_computed
Has a Jacobian been computed (and can therefore be re-used if required)? Default: false.
Definition: problem.h:622
bool Bypass_increase_in_dof_check_during_pruning
Boolean to bypass check of increase in dofs during pruning.
Definition: problem.h:967
virtual void get_dvaluesdt(DoubleVector &f)
Get the time derivative of all values (using get_inverse_mass_matrix_times_residuals(....
Definition: problem.cc:3770
void initialise_dt(const double &dt)
Set all timesteps to the same value, dt, and assign weights for all timesteppers in the problem.
Definition: problem.cc:13424
void add_to_dofs(const double &lambda, const DoubleVector &increment_dofs)
Add lambda x incremenet_dofs[l] to the l-th dof.
Definition: problem.cc:3650
double Timestep_reduction_factor_after_nonconvergence
What it says: If temporally adaptive Newton solver fails to to converge, reduce timestep by this fact...
Definition: problem.h:2331
Vector< double > Dof_current
Storage for the present values of the variables.
Definition: problem.h:777
double Desired_proportion_of_arc_length
Proportion of the arc-length to taken by the parameter.
Definition: problem.h:734
AssemblyHandler *& assembly_handler_pt()
Return a pointer to the assembly handler object.
Definition: problem.h:1570
virtual void actions_after_implicit_timestep_and_error_estimation()
Actions that should be performed after each implicit time step. This is needed if your actions_after_...
Definition: problem.h:1075
void disable_mass_matrix_reuse()
Turn off recyling of the mass matrix in explicit timestepping schemes.
Definition: problem.cc:12025
EigenSolver * Default_eigen_solver_pt
Pointer to the default eigensolver.
Definition: problem.h:191
unsigned Nnewton_iter_taken
Actual number of Newton iterations taken during the most recent iteration.
Definition: problem.h:603
double * bifurcation_parameter_pt() const
Return pointer to the parameter that is used in the bifurcation detection. If we are not tracking a b...
Definition: problem.cc:10274
void copy_haloed_eqn_numbers_helper(const bool &do_halos, const bool &do_external_halos)
A private helper function to copy the haloed equation numbers into the halo equation numbers,...
Definition: problem.cc:17144
virtual void sparse_assemble_row_or_column_compressed_with_vectors_of_pairs(Vector< int * > &column_or_row_index, Vector< int * > &row_or_column_start, Vector< double * > &value, Vector< unsigned > &nnz, Vector< double * > &residual, bool compressed_row_flag)
Private helper function that is used to assemble the Jacobian matrix in the case when the storage is ...
Definition: problem.cc:5322
void send_data_to_be_sent_during_load_balancing(Vector< int > &send_n, Vector< double > &send_data, Vector< int > &send_displacement)
Load balance helper routine: Send data to other processors during load balancing.
Definition: problem.cc:19142
bool Time_adaptive_newton_crash_on_solve_fail
Bool to specify what to do if a Newton solve fails within a time adaptive solve. Default (false) is t...
Definition: problem.h:615
Problem()
Constructor: Allocate space for one time stepper and set all pointers to NULL and set defaults for al...
Definition: problem.cc:69
bool is_dparameter_calculated_analytically(double *const &parameter_pt)
Function to determine whether the parameter derivatives are calculated analytically.
Definition: problem.h:277
void flush_sub_meshes()
Flush the problem's collection of sub-meshes. Must be followed by call to rebuild_global_mesh().
Definition: problem.h:1339
bool & use_predictor_values_as_initial_guess()
Definition: problem.h:2123
unsigned Sparse_assembly_method
Flag to determine which sparse assembly method to use By default we use assembly by vectors of pairs.
Definition: problem.h:641
Time *& time_pt()
Return a pointer to the global time object.
Definition: problem.h:1504
void calculate_predictions()
Calculate predictions.
Definition: problem.cc:11849
Vector< unsigned > Last_el_plus_one_for_assembly
Last element (plus one) to be assembled by given processor for non-distributed problem (only kept up ...
Definition: problem.h:524
double * global_dof_pt(const unsigned &i)
Return a pointer to the dof, indexed by global equation number which may be haloed or stored locally....
Definition: problem.h:1768
virtual void actions_after_read_unstructured_meshes()
Actions that are to be performed before reading in restart data for problems involving unstructured b...
Definition: problem.h:1109
void copy(Problem *orig_problem_pt)
Copy Data values, nodal positions etc from specified problem. Note: This is not a copy constructor....
Definition: problem.cc:12058
virtual void get_jacobian(DoubleVector &residuals, DenseDoubleMatrix &jacobian)
Return the fully-assembled Jacobian and residuals for the problem Interface for the case when the Jac...
Definition: problem.cc:3921
void set_explicit_time_stepper_pt(ExplicitTimeStepper *const &explicit_time_stepper_pt)
Set the explicit timestepper for the problem. The function will automatically create or resize the Ti...
Definition: problem.cc:1672
virtual void actions_after_distribute()
Actions to be performed after a (mesh) distribution.
Definition: problem.h:1119
virtual void actions_before_implicit_timestep()
Actions that should be performed before each implicit time step. This is needed when one wants to sol...
Definition: problem.h:1064
LinearSolver * Linear_solver_pt
Pointer to the linear solver for the problem.
Definition: problem.h:173
bool Doc_time_in_distribute
Protected boolean flag to provide comprehensive timimings during problem distribution....
Definition: problem.h:637
unsigned Max_newton_iterations
Maximum number of Newton iterations.
Definition: problem.h:599
Vector< Vector< unsigned > > Sparse_assemble_with_arrays_previous_allocation
the number of elements in each row of a compressed matrix in the previous matrix assembly.
Definition: problem.h:667
virtual void actions_after_parameter_increase(double *const &parameter_pt)
Empty virtual function; provides hook to perform actions after the increase in the arclength paramete...
Definition: problem.h:1161
friend class HopfHandler
Definition: problem.h:155
friend class PitchForkHandler
Definition: problem.h:154
void calculate_continuation_derivatives(double *const &parameter_pt)
A function to calculate the derivatives wrt the arc-length. This version of the function actually doe...
Definition: problem.cc:9908
void store_current_dof_values()
Store the current values of the degrees of freedom.
Definition: problem.cc:8787
bool Problem_has_been_distributed
Has the problem been distributed amongst multiple processors?
Definition: problem.h:964
void synchronise_all_dofs()
Perform all required synchronisation in solvers.
Definition: problem.cc:16632
bool Empty_actions_before_read_unstructured_meshes_has_been_called
Boolean to indicate that empty actions_before_read_unstructured_meshes() function has been called.
Definition: problem.h:218
OomphCommunicator * communicator_pt()
access function to the oomph-lib communicator
Definition: problem.h:1246
bool Mass_matrix_has_been_computed
Has the mass matrix been computed (and can therefore be reused) Default: false.
Definition: problem.h:695
unsigned nglobal_data() const
Return the number of global data values.
Definition: problem.h:1690
virtual void actions_before_adapt()
Actions that are to be performed before a mesh adaptation. These might include removing any additiona...
Definition: problem.h:1022
void newton_solve()
Use Newton method to solve the problem.
Definition: problem.cc:8976
bool First_jacobian_sign_change
Boolean to indicate whether a sign change has occured in the Jacobian.
Definition: problem.h:796
void refine_distributed_base_mesh(Vector< Vector< Vector< unsigned >>> &to_be_refined_on_each_root, const unsigned &max_level_overall)
Load balance helper routine: refine each new base (sub)mesh based upon the elements to be refined wit...
Definition: problem.cc:20183
void calculate_continuation_derivatives_fd_helper(double *const &parameter_pt)
A function that performs the guts of the continuation derivative calculation in arc-length continuati...
Definition: problem.cc:10197
double Continuation_direction
The direction of the change in parameter that will ensure that a branch is followed in one direction ...
Definition: problem.h:749
unsigned Parallel_sparse_assemble_previous_allocation
The amount of data allocated during the previous parallel sparse assemble. Used to optimise the next ...
Definition: problem.h:960
void enable_mass_matrix_reuse()
Enable recycling of the mass matrix in explicit timestepping schemes. Useful for timestepping on fixe...
Definition: problem.cc:12000
void steady_newton_solve(unsigned const &max_adapt=0)
Solve a steady problem using adaptive Newton's method, but in the context of an overall unstady probl...
Definition: problem.cc:9485
bool Scale_arc_length
Boolean to control whether arc-length should be scaled.
Definition: problem.h:731
double doubly_adaptive_unsteady_newton_solve_helper(const double &dt, const double &epsilon, const unsigned &max_adapt, const unsigned &suppress_resolve_after_spatial_adapt, const bool &first, const bool &shift=true)
Private helper function that actually performs the unsteady "doubly" adaptive Newton solve....
Definition: problem.cc:11572
bool Empty_actions_after_read_unstructured_meshes_has_been_called
Boolean to indicate that empty actions_after_read_unstructured_meshes() function has been called.
Definition: problem.h:222
virtual void get_residuals(DoubleVector &residuals)
Return the fully-assembled residuals Vector for the problem: Virtual so it can be overloaded in for m...
Definition: problem.cc:3800
Vector< GeneralisedElement * > Base_mesh_element_pt
Vector to store the correspondence between a root element and its element number within the global me...
Definition: problem.h:548
double & dof(const unsigned &i)
i-th dof in the problem
Definition: problem.h:1817
void get_fd_jacobian(DoubleVector &residuals, DenseMatrix< double > &jacobian)
Return the fully-assembled Jacobian and residuals, generated by finite differences.
Definition: problem.cc:7800
virtual void sparse_assemble_row_or_column_compressed_with_two_vectors(Vector< int * > &column_or_row_index, Vector< int * > &row_or_column_start, Vector< double * > &value, Vector< unsigned > &nnz, Vector< double * > &residual, bool compressed_row_flag)
Private helper function that is used to assemble the Jacobian matrix in the case when the storage is ...
Definition: problem.cc:5675
virtual Problem * make_copy()
Make and return a pointer to the copy of the problem. A virtual function that must be filled in by th...
Definition: problem.cc:12204
double Maximum_dt
Maximum desired dt.
Definition: problem.h:709
unsigned long set_timestepper_for_all_data(TimeStepper *const &time_stepper_pt, const bool &preserve_existing_data=false)
Set all problem data to have the same timestepper (timestepper_pt) Return the new number of dofs in t...
Definition: problem.cc:11765
void activate_pitchfork_tracking(double *const &parameter_pt, const DoubleVector &symmetry_vector, const bool &block_solve=true)
Turn on pitchfork tracking using the augmented system specified in the PitchForkHandler class....
Definition: problem.cc:10381
virtual void dump(std::ofstream &dump_file) const
Dump refinement pattern of all refineable meshes and all generic Problem data to file for restart.
Definition: problem.cc:12222
bool Use_finite_differences_for_continuation_derivatives
Boolean to specify which scheme to use to calculate the continuation derivatievs.
Definition: problem.h:803
bool Arc_length_step_taken
Boolean to indicate whether an arc-length step has been taken.
Definition: problem.h:799
void set_pinned_values_to_zero()
Set all pinned values to zero. Used to set boundary conditions to be homogeneous in the copy of the p...
Definition: problem.cc:4281
bool Default_set_initial_condition_called
Has default set_initial_condition function been called? Default: false.
Definition: problem.h:211
void get_bifurcation_eigenfunction(Vector< DoubleVector > &eigenfunction)
Return the eigenfunction calculated as part of a bifurcation tracking process. If we are not tracking...
Definition: problem.cc:10284
double Relaxation_factor
Relaxation fator for Newton method (only a fractional Newton correction is applied if this is less th...
Definition: problem.h:592
void add_time_stepper_pt(TimeStepper *const &time_stepper_pt)
Add a timestepper to the problem. The function will automatically create or resize the Time object so...
Definition: problem.cc:1631
void refine_selected_elements(const Vector< unsigned > &elements_to_be_refined)
Refine (one and only!) mesh by splitting the elements identified by their numbers relative to the pro...
Definition: problem.cc:15091
void prune_halo_elements_and_nodes(DocInfo &doc_info, const bool &report_stats)
(Irreversibly) prune halo(ed) elements and nodes, usually after another round of refinement,...
Definition: problem.cc:1139
void solve_eigenproblem_legacy(const unsigned &n_eval, Vector< std::complex< double >> &eigenvalue, Vector< DoubleVector > &eigenvector, const bool &steady=true)
Get derivative of an element in the problem wrt a global parameter, used in continuation problems.
Definition: problem.cc:8293
virtual void get_inverse_mass_matrix_times_residuals(DoubleVector &Mres)
Return the residual vector multiplied by the inverse mass matrix Virtual so that it can be overloaded...
Definition: problem.cc:3665
void check_halo_schemes()
Check the halo/haloed node/element schemes.
Definition: problem.h:2245
double Parameter_current
Storage for the present value of the global parameter.
Definition: problem.h:755
void assign_eigenvector_to_dofs(DoubleVector &eigenvector)
Assign the eigenvector passed to the function to the dofs in the problem so that it can be output by ...
Definition: problem.cc:8896
@ Uniform_matrix_distribution
Definition: problem.h:831
@ Default_matrix_distribution
Definition: problem.h:829
@ Problem_matrix_distribution
Definition: problem.h:830
unsigned setup_element_count_per_dof()
Function that populates the Element_counter_per_dof vector with the number of elements that contribut...
Definition: problem.cc:230
LinearSolver *& mass_matrix_solver_for_explicit_timestepper_pt()
Return a pointer to the linear solver object used for explicit time stepping.
Definition: problem.h:1479
OomphCommunicator * Communicator_pt
The communicator for this problem.
Definition: problem.h:1242
double Newton_solver_tolerance
The Tolerance below which the Newton Method is deemed to have converged.
Definition: problem.h:596
void get_flat_packed_refinement_pattern_for_load_balancing(const Vector< unsigned > &old_domain_for_base_element, const Vector< unsigned > &new_domain_for_base_element, const unsigned &max_refinement_level_overall, std::map< unsigned, Vector< unsigned >> &flat_packed_refinement_info_for_root)
Get flat-packed refinement pattern for each root element in current mesh (labeled by unique number of...
Definition: problem.cc:20028
void set_consistent_pinned_values_for_continuation()
Private helper function that is used to set the appropriate pinned values for continuation.
Definition: problem.cc:10659
void activate_bifurcation_tracking(double *const &parameter_pt, const DoubleVector &eigenvector, const bool &block_solve=true)
Activate generic bifurcation tracking for a single (real) eigenvalue where the initial guess for the ...
Definition: problem.cc:10322
bool Discontinuous_element_formulation
Is the problem a discontinuous one, i.e. can the elemental contributions be treated independently....
Definition: problem.h:700
bool Doc_imbalance_in_parallel_assembly
Boolean to switch on assessment of load imbalance in parallel assembly of distributed problem.
Definition: problem.h:510
long synchronise_eqn_numbers(const bool &assign_local_eqn_numbers=true)
Classify any non-classified nodes into halo/haloed and synchronise equation numbers....
Definition: problem.cc:16898
void create_new_linear_algebra_distribution(LinearAlgebraDistribution *&dist_pt)
Get new linear algebra distribution (you're in charge of deleting it!)
Definition: problem.cc:299
void p_refine_uniformly_aux(const Vector< unsigned > &nrefine_for_mesh, DocInfo &doc_info, const bool &prune)
Helper function to do compund p-refinement of (all) p-refineable (sub)mesh(es) uniformly as many time...
Definition: problem.cc:15785
void calculate_continuation_derivatives_helper(const DoubleVector &z)
A function that performs the guts of the continuation derivative calculation in arc length continuati...
Definition: problem.cc:10105
Vector< Problem * > Copy_of_problem_pt
Vector of pointers to copies of the problem used in adaptive bifurcation tracking problems (ALH: TEMP...
Definition: problem.h:234
Vector< unsigned > distribute(const Vector< unsigned > &element_partition, DocInfo &doc_info, const bool &report_stats=false)
Distribute the problem and doc, using the specified partition; returns a vector which details the par...
Definition: problem.cc:490
Mesh * Mesh_pt
The mesh pointer.
Definition: problem.h:167
double Parameter_derivative
Storage for the derivative of the global parameter wrt arc-length.
Definition: problem.h:752
void add_eigenvector_to_dofs(const double &epsilon, const DoubleVector &eigenvector)
Add the eigenvector passed to the function scaled by the constat epsilon to the dofs in the problem s...
Definition: problem.cc:8933
Vector< Data * > Global_data_pt
Vector of global data: "Nobody" (i.e. none of the elements etc.) is "in charge" of this Data so it wo...
Definition: problem.h:425
void recompute_load_balanced_assembly()
Helper function to re-assign the first and last elements to be assembled by each processor during par...
Definition: problem.cc:1775
Vector< double * > Dof_pt
Vector of pointers to dofs.
Definition: problem.h:554
void p_adapt()
p-adapt problem: Perform mesh adaptation for (all) refineable (sub)mesh(es), based on their own error...
Definition: problem.h:3010
double DTSF_max_increase
Maximum possible increase of dt between time-steps in adaptive schemes.
Definition: problem.h:713
bool Must_recompute_load_balance_for_assembly
Boolean indicating that the division of elements over processors during the assembly process must be ...
Definition: problem.h:529
virtual void shift_time_values()
Shift all values along to prepare for next timestep.
Definition: problem.cc:11827
Mesh *& mesh_pt()
Return a pointer to the global mesh.
Definition: problem.h:1280
void set_default_first_and_last_element_for_assembly()
Set default first and last elements for parallel assembly of non-distributed problem.
Definition: problem.cc:1699
bool Keep_temporal_error_below_tolerance
Boolean to decide if a timestep is to be rejected if the error estimate post-solve (computed by globa...
Definition: problem.h:2338
bool Shut_up_in_newton_solve
Boolean to indicate if all output is suppressed in Problem::newton_solve(). Defaults to false.
Definition: problem.h:2320
void set_dofs(const DoubleVector &dofs)
Set the values of the dofs.
Definition: problem.cc:3497
void setup_base_mesh_info_after_pruning()
Helper function to re-setup the Base_mesh enumeration (used during load balancing) after pruning.
Definition: problem.cc:20295
Vector< Mesh * > Sub_mesh_pt
Vector of pointers to submeshes.
Definition: problem.h:170
Vector< double > Max_res
Maximum residuals at start and after each newton iteration.
Definition: problem.h:606
EigenSolver * Eigen_solver_pt
Pointer to the eigen solver for the problem.
Definition: problem.h:182
bool Bisect_to_find_bifurcation
Boolean to control wheter bisection is used to located bifurcation.
Definition: problem.h:793
void explicit_timestep(const double &dt, const bool &shift_values=true)
Take an explicit timestep of size dt and optionally shift any stored values of the time history.
Definition: problem.cc:11111
void delete_all_external_storage()
Wrapper function to delete external storage for each submesh of the problem.
Definition: problem.cc:16548
friend class BlockPitchForkLinearSolver
Definition: problem.h:159
double DTSF_min_decrease
Minimum allowed decrease of dt between time-steps in adaptive schemes. Lower scaling values will reje...
Definition: problem.h:718
virtual void symmetrise_eigenfunction_for_adaptive_pitchfork_tracking()
Virtual function that is used to symmetrise the problem so that the current solution exactly satisfie...
Definition: problem.cc:10252
double Minimum_dt_but_still_proceed
If Minimum_dt_but_still_proceed positive, then dt will not be reduced below this value during adaptiv...
Definition: problem.h:725
double adaptive_unsteady_newton_solve(const double &dt_desired, const double &epsilon)
Attempt to advance timestep by dt_desired. If the solution fails the timestep will be halved until co...
Definition: problem.cc:11249
unsigned Desired_newton_iterations_ds
The desired number of Newton Steps to reach convergence at each step along the arc.
Definition: problem.h:784
void rebuild_global_mesh()
If one of the submeshes has changed (e.g. by mesh adaptation) we need to update the global mesh....
Definition: problem.cc:1619
ExplicitTimeStepper *& explicit_time_stepper_pt()
Return a pointer to the explicit timestepper.
Definition: problem.h:1555
void activate_hopf_tracking(double *const &parameter_pt, const bool &block_solve=true)
Turn on Hopf bifurcation tracking using the augmented system specified in the HopfHandler class....
Definition: problem.cc:10411
Vector< double * > Halo_dof_pt
Storage for the halo degrees of freedom (only required) when accessing via the global equation number...
Definition: problem.h:581
void deactivate_bifurcation_tracking()
Deactivate all bifuraction tracking, by reseting the assembly handler to the default.
Definition: problem.h:2482
void globally_convergent_line_search(const Vector< double > &x_old, const double &half_residual_squared_old, DoubleVector &gradient, DoubleVector &newton_dir, double &half_residual_squared, const double &stpmax)
Line search helper for globally convergent Newton method.
Definition: problem.cc:9339
void get_hessian_vector_products(DoubleVectorWithHaloEntries const &Y, Vector< DoubleVectorWithHaloEntries > const &C, Vector< DoubleVectorWithHaloEntries > &product)
Return the product of the global hessian (derivative of Jacobian matrix with respect to all variables...
Definition: problem.cc:7962
virtual double global_temporal_error_norm()
Function to calculate a global error norm, used in adaptive timestepping to control the change in tim...
Definition: problem.h:1230
@ Perform_assembly_using_two_arrays
Definition: problem.h:650
@ Perform_assembly_using_maps
Definition: problem.h:648
@ Perform_assembly_using_two_vectors
Definition: problem.h:647
@ Perform_assembly_using_vectors_of_pairs
Definition: problem.h:646
@ Perform_assembly_using_lists
Definition: problem.h:649
int Sign_of_jacobian
Storage for the sign of the global Jacobian.
Definition: problem.h:745
std::map< GeneralisedElement *, unsigned > Base_mesh_element_number_plus_one
Map which stores the correspondence between a root element and its element number (plus one) within t...
Definition: problem.h:538
double Max_residuals
Maximum desired residual: if the maximum residual exceeds this value, the program will exit.
Definition: problem.h:610
unsigned nsub_mesh() const
Return number of submeshes.
Definition: problem.h:1323
double & time()
Return the current value of continuous time.
Definition: problem.cc:11724
unsigned self_test()
Self-test: Check meshes and global data. Return 0 for OK.
Definition: problem.cc:13469
unsigned ntime_stepper() const
Return the number of time steppers.
Definition: problem.h:1684
void activate_fold_tracking(double *const &parameter_pt, const bool &block_solve=true)
Turn on fold tracking using the augmented system specified in the FoldHandler class....
Definition: problem.cc:10296
bool Use_globally_convergent_newton_method
Use the globally convergent newton method.
Definition: problem.h:214
LinearSolver * Default_linear_solver_pt
Pointer to the default linear solver.
Definition: problem.h:188
double Minimum_ds
Minimum desired value of arc-length.
Definition: problem.h:787
void get_data_to_be_sent_during_load_balancing(const Vector< unsigned > &element_domain_on_this_proc, Vector< int > &send_n, Vector< double > &send_data, Vector< int > &send_displacement, Vector< unsigned > &old_domain_for_base_element, Vector< unsigned > &new_domain_for_base_element, unsigned &max_refinement_level_overall)
Load balance helper routine: Get data to be sent to other processors during load balancing and other ...
Definition: problem.cc:19472
void restore_dof_values()
Restore the stored values of the degrees of freedom.
Definition: problem.cc:8833
double Numerical_zero_for_sparse_assembly
A tolerance used to determine whether the entry in a sparse matrix is zero. If it is then storage nee...
Definition: problem.h:671
double FD_step_used_in_get_hessian_vector_products
Definition: problem.h:685
virtual void partition_global_mesh(Mesh *&global_mesh_pt, DocInfo &doc_info, Vector< unsigned > &element_domain, const bool &report_stats=false)
Partition the global mesh, return vector specifying the processor number for each element....
Definition: problem.cc:964
unsigned Sparse_assemble_with_arrays_initial_allocation
the number of elements to initially allocate for a matrix row within the sparse_assembly_with_two_arr...
Definition: problem.h:658
void bifurcation_adapt_doc_errors(const unsigned &bifurcation_type)
A function that is used to document the errors used in the adaptive solution of bifurcation problems.
Definition: problem.cc:13841
double Ds_current
Storage for the current step value.
Definition: problem.h:780
virtual void set_initial_condition()
Set initial condition (incl previous timesteps). We need to establish this interface because I....
Definition: problem.h:1198
LinearSolver * Mass_matrix_solver_for_explicit_timestepper_pt
Pointer to the linear solver used for explicit time steps (this is likely to be different to the line...
Definition: problem.h:179
void get_all_halo_data(std::map< unsigned, double * > &map_of_halo_data)
Get pointers to all possible halo data indexed by global equation number in a map.
Definition: problem.cc:16570
void load_balance()
Balance the load of a (possibly non-uniformly refined) problem that has already been distributed,...
Definition: problem.h:1382
double & dof_derivative(const unsigned &i)
Access function to the derivative of the i-th (local) dof with respect to the arc length,...
Definition: problem.h:1168
double Minimum_dt
Minimum desired dt: if dt falls below this value, exit.
Definition: problem.h:706
TimeStepper *& time_stepper_pt()
Access function for the pointer to the first (presumably only) timestepper.
Definition: problem.h:1524
double arc_length_step_solve(double *const &parameter_pt, const double &ds, const unsigned &max_adapt=0)
Solve a steady problem using arc-length continuation, when the parameter that becomes a variable corr...
Definition: problem.cc:10487
virtual void actions_after_adapt()
Actions that are to be performed after a mesh adaptation.
Definition: problem.h:1025
void p_refine_uniformly()
p-refine (all) p-refineable (sub)mesh(es) uniformly and rebuild problem
Definition: problem.h:2866
bool Use_continuation_timestepper
Boolean to control original or new storage of dof stuff.
Definition: problem.h:758
void calculate_continuation_derivatives_fd(double *const &parameter_pt)
A function to calculate the derivatives with respect to the arc-length required for continuation by f...
Definition: problem.cc:10011
LinearAlgebraDistribution * Dof_distribution_pt
The distribution of the DOFs in this problem. This object is created in the Problem constructor and s...
Definition: problem.h:460
void refine_uniformly()
Refine (all) refineable (sub)mesh(es) uniformly and rebuild problem.
Definition: problem.h:2769
static ContinuationStorageScheme Continuation_time_stepper
Storage for the single static continuation timestorage object.
Definition: problem.h:761
bool Problem_is_nonlinear
Boolean flag indicating if we're dealing with a linear or nonlinear Problem – if set to false the New...
Definition: problem.h:628
virtual void sparse_assemble_row_or_column_compressed_with_maps(Vector< int * > &column_or_row_index, Vector< int * > &row_or_column_start, Vector< double * > &value, Vector< unsigned > &nnz, Vector< double * > &residual, bool compressed_row_flag)
Private helper function that is used to assemble the Jacobian matrix in the case when the storage is ...
Definition: problem.cc:4561
void send_refinement_info_helper(Vector< unsigned > &old_domain_for_base_element, Vector< unsigned > &new_domain_for_base_element, const unsigned &max_refinement_level_overall, std::map< unsigned, Vector< unsigned >> &refinement_info_for_root_local, Vector< Vector< Vector< unsigned >>> &refinement_info_for_root_elements)
Send refinement information between processors.
Definition: problem.cc:18521
AssemblyHandler * Default_assembly_handler_pt
Pointer to the default assembly handler.
Definition: problem.h:194
void get_my_eqns(AssemblyHandler *const &assembly_handler_pt, const unsigned &el_lo, const unsigned &el_hi, Vector< unsigned > &my_eqns)
Helper method that returns the (unique) global equations to which the elements in the range el_lo to ...
Definition: problem.cc:6486
virtual void read(std::ifstream &restart_file, bool &unsteady_restart)
Read refinement pattern of all refineable meshes and refine them accordingly, then read all Data and ...
Definition: problem.cc:12444
virtual ~Problem()
Virtual destructor to clean up memory.
Definition: problem.cc:181
void get_dofs(DoubleVector &dofs) const
Return the vector of dofs, i.e. a vector containing the current values of all unknowns.
Definition: problem.cc:2565
Time * Time_pt
Pointer to global time for the problem.
Definition: problem.h:197
DoubleVectorWithHaloEntries Element_count_per_dof
Counter that records how many elements contribute to each dof. Used to determine the (discrete) arc-l...
Definition: problem.h:560
Data *& global_data_pt(const unsigned &i)
Return a pointer to the the i-th global data object.
Definition: problem.h:1647
virtual void actions_before_newton_convergence_check()
Any actions that are to be performed before the residual is checked in the Newton method,...
Definition: problem.h:1048
unsigned newton_solve_continuation(double *const &parameter_pt)
Perform a basic arc-length continuation step using Newton's method. Returns number of Newton steps ta...
Definition: problem.cc:9569
double Max_permitted_error_for_halo_check
Threshold for error throwing in Problem::check_halo_schemes()
Definition: problem.h:2304
unsigned Sparse_assemble_with_arrays_allocation_increment
the number of elements to add to a matrix row when the initial allocation is exceeded within the spar...
Definition: problem.h:663
void reset_assembly_handler_to_default()
Reset the system to the standard non-augemented state.
Definition: problem.cc:10468
virtual void actions_after_newton_step()
Any actions that are to be performed after each individual Newton step. Most likely to be used for di...
Definition: problem.h:1058
bool Pause_at_end_of_sparse_assembly
Protected boolean flag to halt program execution during sparse assemble process to assess peak memory...
Definition: problem.h:633
void remove_null_pointers_from_external_halo_node_storage()
Consolidate external halo node storage by removing nulled out pointers in external halo and haloed sc...
Definition: problem.cc:3264
Vector< double > * Saved_dof_pt
Pointer to vector for backup of dofs.
Definition: problem.h:207
void unsteady_newton_solve(const double &dt)
Advance time by dt and solve by Newton's method. This version always shifts time values.
Definition: problem.cc:11146
AssemblyHandler * Assembly_handler_pt
Definition: problem.h:185
void get_all_error_estimates(Vector< Vector< double >> &elemental_error)
Return the error estimates computed by (all) refineable (sub)mesh(es) in the elemental_error structur...
Definition: problem.cc:14820
virtual void actions_before_read_unstructured_meshes()
Actions that are to be performed before reading in restart data for problems involving unstructured b...
Definition: problem.h:1095
Vector< TimeStepper * > Time_stepper_pt
The Vector of time steppers (there could be many different ones in multiphysics problems)
Definition: problem.h:201
void doc_errors()
Get max and min error for all elements in submeshes.
Definition: problem.h:3055
bool Mass_matrix_reuse_is_enabled
Is re-use of the mass matrix in explicit timestepping enabled Default:false.
Definition: problem.h:691
void get_derivative_wrt_global_parameter(double *const &parameter_pt, DoubleVector &result)
Get the derivative of the entire residuals vector wrt a global parameter, used in continuation proble...
Definition: problem.cc:7864
void adapt_based_on_error_estimates(unsigned &n_refined, unsigned &n_unrefined, Vector< Vector< double >> &elemental_error)
Adapt problem: Perform mesh adaptation for (all) refineable (sub)mesh(es), based on the error estimat...
Definition: problem.cc:14721
bool distributed() const
If we have MPI return the "problem has been distributed" flag, otherwise it can't be distributed so r...
Definition: problem.h:808
bool are_hessian_products_calculated_analytically()
Function to determine whether the hessian products are calculated analytically.
Definition: problem.h:303
bool does_pointer_correspond_to_problem_data(double *const &parameter_pt)
Return a boolean flag to indicate whether the pointer parameter_pt refers to values stored in a Data ...
Definition: problem.cc:10039
ExplicitTimeStepper * Explicit_time_stepper_pt
Pointer to a single explicit timestepper.
Definition: problem.h:204
double arc_length_step_solve_helper(double *const &parameter_pt, const double &ds, const unsigned &max_adapt)
Private helper function that actually contains the guts of the arc-length stepping,...
Definition: problem.cc:10708
void solve_eigenproblem(const unsigned &n_eval, Vector< std::complex< double >> &alpha, Vector< double > &beta, Vector< DoubleVector > &eigenvector_real, Vector< DoubleVector > &eigenvector_imag, const bool &steady=true)
Solve an eigenproblem as assembled by the Problem's constituent elements. Calculate (at least) n_eval...
Definition: problem.cc:8348
bool Use_predictor_values_as_initial_guess
Use values from the time stepper predictor as an initial guess.
Definition: problem.h:229
RefineableElements are FiniteElements that may be subdivided into children to provide a better local ...
Tree * tree_pt()
Access function: Pointer to quadtree representation of this element.
virtual RefineableElement * root_element_pt()
Pointer to the root element in refinement hierarchy (must be implemented in specific elements that do...
Base class for refineable meshes. Provides standardised interfaces for the following standard mesh ad...
General SolidMesh class.
Definition: mesh.h:2562
A Class for nodes that deform elastically (i.e. position is an unknown in the problem)....
Definition: nodes.h:1686
void copy(SolidNode *orig_node_pt)
Copy nodal positions and associated data from specified node object.
Definition: nodes.cc:3433
Data *const & variable_position_pt() const
Pointer to variable_position data (const version)
Definition: nodes.h:1765
bool position_is_pinned(const unsigned &i)
Test whether the i-th coordinate is pinned, 0: false; 1: true.
Definition: nodes.h:1803
/////////////////////////////////////////////////////////////////////// /////////////////////////////...
Definition: spines.h:613
//////////////////////////////////////////////////////////////////////////// ////////////////////////...
////////////////////////////////////////////////////////////////////// //////////////////////////////...
Definition: timesteppers.h:231
virtual unsigned ndt() const =0
Number of timestep increments that are required by the scheme.
virtual void set_weights()=0
Function to set the weights for present timestep (don't need to pass present timestep or previous tim...
virtual void set_predictor_weights()
Set the weights for the predictor previous timestep (currently empty – overwrite for specific scheme)
Definition: timesteppers.h:630
virtual void actions_before_timestep(Problem *problem_pt)
Interface for any actions that need to be performed before a time step.
Definition: timesteppers.h:662
virtual void actions_after_timestep(Problem *problem_pt)
Interface for any actions that need to be performed after a time step.
Definition: timesteppers.h:666
void make_steady()
Function to make the time stepper temporarily steady. This is trivially achieved by setting all the w...
Definition: timesteppers.h:374
virtual void undo_make_steady()
Reset the is_steady status of a specific TimeStepper to its default and re-assign the weights.
Definition: timesteppers.h:482
ExplicitTimeStepper * explicit_predictor_pt()
Get the pointer to the explicit timestepper to use as a predictor in adaptivity if Predict_by_explici...
Definition: timesteppers.h:403
void update_predicted_time(const double &new_time)
Set the time that the current predictions apply for, only needed for paranoid checks when doing Predi...
Definition: timesteppers.h:417
bool is_steady() const
Flag to indicate if a timestepper has been made steady (possibly temporarily to switch off time-depen...
Definition: timesteppers.h:389
virtual void set_error_weights()
Set the weights for the error computation, (currently empty – overwrite for specific scheme)
Definition: timesteppers.h:642
Time *const & time_pt() const
Access function for the pointer to time (const version)
Definition: timesteppers.h:572
Class to keep track of discrete/continous time. It is essential to have a single Time object when usi...
Definition: timesteppers.h:63
double & dt(const unsigned &t=0)
Return the value of the t-th stored timestep (t=0: present; t>0: previous).
Definition: timesteppers.h:136
double & time()
Return the current value of the continuous time.
Definition: timesteppers.h:123
void shift_dt()
Update all stored values of dt by shifting each value along the array. This function must be called b...
Definition: timesteppers.h:174
void initialise_dt(const double &dt_)
Set all timesteps to the same value, dt.
Definition: timesteppers.h:99
unsigned ndt() const
Return the number of timesteps stored.
Definition: timesteppers.h:129
void resize(const unsigned &n_dt)
Resize the vector holding the number of previous timesteps and initialise the new values to zero.
Definition: timesteppers.h:93
///////////////////////////////////////////////////////////////// ///////////////////////////////////...
void refine_base_mesh(Vector< Vector< unsigned >> &to_be_refined)
Refine base mesh according to specified refinement pattern.
unsigned uniform_refinement_level_when_pruned() const
Level to which the mesh was uniformly refined when it was pruned (const version)
virtual void get_refinement_levels(unsigned &min_refinement_level, unsigned &max_refinement_level)
Get max/min refinement levels in mesh.
void refine_uniformly(DocInfo &doc_info)
Refine mesh uniformly and doc process.
TreeRoot is a Tree that forms the root of a (recursive) tree. The "root node" is special as it holds ...
Definition: tree.h:324
void stick_all_tree_nodes_into_vector(Vector< Tree * > &)
Traverse and stick pointers to all "nodes" into Vector.
Definition: tree.cc:277
RefineableElement * object_pt() const
Return the pointer to the object (RefineableElement) represented by the tree.
Definition: tree.h:88
void stick_leaves_into_vector(Vector< Tree * > &)
Traverse tree and stick pointers to leaf "nodes" (only) into Vector.
Definition: tree.cc:255
TreeRoot *& root_pt()
Return pointer to root of the tree.
Definition: tree.h:141
Base class for triangle meshes (meshes made of 2D triangle elements). Note: we choose to template Tri...
Definition: triangle_mesh.h:52
void dump_triangulateio(std::ostream &dump_file)
Dump the triangulateio structure to a dump file and record boundary coordinates of boundary nodes.
virtual void read_distributed_info_for_restart(std::istream &restart_file)
Virtual function that is used to read info. related with distributed triangle meshes.
bool use_triangulateio_restart() const
const access for Use_triangulateio_restart.
void remesh_from_triangulateio(std::istream &restart_file)
Regenerate the mesh from a dumped triangulateio file and dumped boundary coordinates of boundary node...
virtual void dump_distributed_info_for_restart(std::ostream &dump_file)
Virtual function that is used to dump info. related with distributed triangle meshes.
virtual void reestablish_distribution_info_for_restart(OomphCommunicator *comm_pt, std::istream &restart_file)
Virtual function used to re-establish any additional info. related with the distribution after a re-s...
void initialise(const _Tp &__value)
Iterate over all values and set to the desired value.
Definition: Vector.h:167
std::string string(const unsigned &i)
Return the i-th string or "" if the relevant string hasn't been defined.
bool Doc_comprehensive_timings
Global boolean to switch on comprehensive timing – can probably be declared const false when developm...
unsigned Max_newton_iterations
Maximum number of newton iterations.
Definition: elements.cc:1654
void partition_distributed_mesh(Problem *problem_pt, const unsigned &objective, Vector< unsigned > &element_domain_on_this_proc, const bool &bypass_metis=false)
Use METIS to assign each element in an already-distributed mesh to a domain. On return,...
void partition_mesh(Problem *problem_pt, const unsigned &ndomain, const unsigned &objective, Vector< unsigned > &element_domain)
Use METIS to assign each element to a domain. On return, element_domain[ielem] contains the number of...
std::string to_string(T object, unsigned float_precision=8)
Conversion function that should work for anything with operator<< defined (at least all basic types).
void clean_up_memory()
Clean up function that deletes anything dynamically allocated in this namespace.
void setup()
Setup terminate helper.
double timer()
returns the time in seconds after some point in past
std::string convert_secs_to_formatted_string(const double &time_in_sec)
Returns a nicely formatted string from an input time in seconds; the format depends on the size of ti...
//////////////////////////////////////////////////////////////////// ////////////////////////////////...
void pause(std::string message)
Pause and display message.
OomphInfo oomph_info
Single (global) instantiation of the OomphInfo object – this is used throughout the library as a "rep...