However, this only works sequentially, if I run on more than one processor it crashes with the error below.
Traceback (most recent call last):
File "driver.py", line 483, in <module>
Traceback (most recent call last):
File "driver.py", line 483, in <module>
main(parameter_filename)
File "driver.py", line 378, in main
main(parameter_filename)
File "driver.py", line 378, in main
u,p,b = gravitywave_solver_matrixfree.solve(r_u,r_p,r_b)
File "/Users/eikemueller/PostDocBath/EllipticSolvers/Firedrake_workspace/firedrake-helmholtzsolver/source/gravitywaves.py", line 208, in solve
self._ksp.solve(self._y,self._x)
File "KSP.pyx", line 353, in petsc4py.PETSc.KSP.solve (src/petsc4py.PETSc.c:139671)
u,p,b = gravitywave_solver_matrixfree.solve(r_u,r_p,r_b)
File "/Users/eikemueller/PostDocBath/EllipticSolvers/Firedrake_workspace/firedrake-helmholtzsolver/source/gravitywaves.py", line 208, in solve
self._ksp.solve(self._y,self._x)
File "KSP.pyx", line 353, in petsc4py.PETSc.KSP.solve (src/petsc4py.PETSc.c:139671)
File "libpetsc4py.pyx", line 1330, in libpetsc4py.PCApply_Python (src/libpetsc4py/libpetsc4py.c:13951)
File "libpetsc4py.pyx", line 1330, in libpetsc4py.PCApply_Python (src/libpetsc4py/libpetsc4py.c:13951)
File "/Users/eikemueller/PostDocBath/EllipticSolvers/Firedrake_workspace/firedrake-helmholtzsolver/source/mixedpreconditioners.py", line 138, in apply
File "/Users/eikemueller/PostDocBath/EllipticSolvers/Firedrake_workspace/firedrake-helmholtzsolver/source/mixedpreconditioners.py", line 138, in apply
self._mixedarray.split(x,u,p)
File "/Users/eikemueller/PostDocBath/EllipticSolvers/Firedrake_workspace/firedrake-helmholtzsolver/source/mixedarray.py", line 81, in split
tmp = v.getSubVector(iset)
File "Vec.pyx", line 752, in petsc4py.PETSc.Vec.getSubVector (src/petsc4py.PETSc.c:90842)
self._mixedarray.split(x,u,p)
File "/Users/eikemueller/PostDocBath/EllipticSolvers/Firedrake_workspace/firedrake-helmholtzsolver/source/mixedarray.py", line 81, in split
tmp = v.getSubVector(iset)
File "Vec.pyx", line 752, in petsc4py.PETSc.Vec.getSubVector (src/petsc4py.PETSc.c:90842)
petsc4py.PETSc.Error: error code 60
[0] KSPSolve() line 572 in /Users/eikemueller/PostDocBath/EllipticSolvers/petsc/src/ksp/ksp/interface/itfunc.c
[0] KSPSolve_GMRES() line 234 in /Users/eikemueller/PostDocBath/EllipticSolvers/petsc/src/ksp/ksp/impls/gmres/gmres.c
[0] KSPInitialResidual() line 63 in /Users/eikemueller/PostDocBath/EllipticSolvers/petsc/src/ksp/ksp/interface/itres.c
[0] KSP_PCApply() line 235 in /Users/eikemueller/PostDocBath/EllipticSolvers/petsc/include/petsc-private/kspimpl.h
[0] PCApply() line 449 in /Users/eikemueller/PostDocBath/EllipticSolvers/petsc/src/ksp/pc/interface/precon.c
[0] VecGetSubVector() line 1343 in /Users/eikemueller/PostDocBath/EllipticSolvers/petsc/src/vec/vec/interface/rvector.c
[0] VecCreateMPIWithArray() line 318 in /Users/eikemueller/PostDocBath/EllipticSolvers/petsc/src/vec/vec/impls/mpi/pbvec.c
[0] PetscSplitOwnership() line 93 in /Users/eikemueller/PostDocBath/EllipticSolvers/petsc/src/sys/utils/psplit.c
[0] Nonconforming object sizes
[0] Sum of local lengths 96000 does not equal global length 47616, my local length 47616
likely a call to VecSetSizes() or MatSetSizes() is wrong.
See
http://www.mcs.anl.gov/petsc/documentation/faq.html#split--------------------------------------------------------------------------
MPI_ABORT was invoked on rank 0 in communicator MPI_COMM_WORLD
with errorcode 1.
NOTE: invoking MPI_ABORT causes Open MPI to kill all MPI processes.
You may or may not see output from other processes, depending on
exactly when Open MPI kills them.
--------------------------------------------------------------------------
petsc4py.PETSc.Error: error code 60
[1] KSPSolve() line 572 in /Users/eikemueller/PostDocBath/EllipticSolvers/petsc/src/ksp/ksp/interface/itfunc.c
[1] KSPSolve_GMRES() line 234 in /Users/eikemueller/PostDocBath/EllipticSolvers/petsc/src/ksp/ksp/impls/gmres/gmres.c
[1] KSPInitialResidual() line 63 in /Users/eikemueller/PostDocBath/EllipticSolvers/petsc/src/ksp/ksp/interface/itres.c
[1] KSP_PCApply() line 235 in /Users/eikemueller/PostDocBath/EllipticSolvers/petsc/include/petsc-private/kspimpl.h
[1] PCApply() line 449 in /Users/eikemueller/PostDocBath/EllipticSolvers/petsc/src/ksp/pc/interface/precon.c
[1] VecGetSubVector() line 1343 in /Users/eikemueller/PostDocBath/EllipticSolvers/petsc/src/vec/vec/interface/rvector.c
[1] VecCreateMPIWithArray() line 318 in /Users/eikemueller/PostDocBath/EllipticSolvers/petsc/src/vec/vec/impls/mpi/pbvec.c
[1] PetscSplitOwnership() line 93 in /Users/eikemueller/PostDocBath/EllipticSolvers/petsc/src/sys/utils/psplit.c
[1] Nonconforming object sizes
[1] Sum of local lengths 96000 does not equal global length 48384, my local length 48384
likely a call to VecSetSizes() or MatSetSizes() is wrong.
See
http://www.mcs.anl.gov/petsc/documentation/faq.html#split