00:00:00 last message repeated 1 times
00:00:00 stkensrv4n.fnal.gov 006148 enstore I SL8500G1MC  dismount VOLUME 0,6,1,11 force => 0,Dismount: Forced dismount of VON240 from   0, 6, 1,11
00:00:00 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC  FINISHED dismount VON240 0,6,1,11 returned ('ok', 0, 'dismount VOLUME 0,6,1,11 force => 0,Dismount: Forced dismount of VON240 from   0, 6, 1,11', '', '')
00:00:00 stkenmvr200a.fnal.gov 005076 root I LTO4_100MV  dismounted VON240 ULTRIUM-TD4 00:00:00 Thread media_thread
00:00:00 stkensrv3n.fnal.gov 020124 root I ENCP  Creating zero length output files.
00:00:00 stkensrv3n.fnal.gov 020124 root I ENCP  Sending /pnfs/cms/WAX/7/PHEDEX2/PCP04/Digi/eg03_jets_2g_pt50170.1/ZippedEVD.63703329.63703340.eg_2x1033PU761_TkMu_2_g133_OSC.eg03_jets_2g_pt50170.zip read request to LM: unique_id: stkensrv3n.fnal.gov-1257400800-20124-0 inputfile: /pnfs/cms/WAX/7/PHEDEX2/PCP04/Digi/eg03_jets_2g_pt50170.1/.(access)(000200000000000000D20960) outputfile: /dev/null
00:00:00 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  MSG_TYPE=ADD_TO_LMQ  request added to pending queue for storage group : cms
00:00:00 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  read Q'd /pnfs/cms/WAX/7/PHEDEX2/PCP04/Digi/eg03_jets_2g_pt50170.1/ZippedEVD.63703329.63703340.eg_2x1033PU761_TkMu_2_g133_OSC.eg03_jets_2g_pt50170.zip (stkensrv3n.fnal.gov-1257400800-20124-0) -> /dev/null : library=CD-LTO4G1 family=cms.cms7.cpio_odc requester:root
00:00:01 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC  PROCESSING listDrives
00:00:01 stkensrv4n.fnal.gov 005843 enstore I SL8500F1MC  PROCESSING listDrives
00:00:01 stkensrv4n.fnal.gov 005852 enstore I STKMC  PROCESSING listDrives
00:00:01 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:00:01 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:00:02 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  read_from_hsm next work on vol=VOP775 mover=LTO4_46.mover requester:root
00:00:02 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  HAVE_BOUND:sending read_from_hsm /pnfs/cms/WAX/7/PHEDEX2/PCP04/Digi/eg03_jets_2g_pt50170.1/ZippedEVD.63703329.63703340.eg_2x1033PU761_TkMu_2_g133_OSC.eg03_jets_2g_pt50170.zip to mover LTO4_46.mover ('131.225.13.174', 7846) DEL_DISM 1
00:00:02 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  MSG_TYPE=ADD_TO_LMQ  request added to work at movers queue for storage group : cms
00:00:02 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  HAVE_BOUND:sending {'vc': {'comment': '<= VO5281 VO7669 VO9276 VO9363 VO9434 VO9484', 'declared': 1231186074.0, 'blocksize': 131072, 'storage_group': 'cms', 'library': 'CD-LTO4G1', 'eod_cookie': '0000_000000000_0001446', 'current_location': '0000_000000000_0000001', 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'full'], 'external_label': 'VOP775', 'wrapper': 'cpio_odc', 'remaining_bytes': 1525791744L, 'sum_mounts': 6, 'sum_rd_access': 1446, 'media_type': 'LTO4', 'last_access': 1257400798.0, 'status': ('ok', None), 'si_time': [0.0, 1235726759.0], 'capacity_bytes': 858993459200L, 'non_del_files': 1481, 'sum_wr_err': 0, 'sum_wr_access': 1445, 'file_family': 'cms7', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.cms7.cpio_odc', 'modification_time': 1238376412.0, 'write_protected': 'y', 'sum_rd_err': 0, 'first_access': 1235693914.0}, 'outfilepath': '/dev/null', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': 1, 'curpri': 1}, 'file_size': 2118453701L, 'mover': 'LTO4_46.mover', 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/cms/WAX/7/PHEDEX2/PCP04/Digi/eg03_jets_2g_pt50170.1/ZippedEVD.63703329.63703340.eg_2x1033PU761_TkMu_2_g133_OSC.eg03_jets_2g_pt50170.zip', 'uid': 0, 'uname': 'root', 'gid': 0, 'rmajor': 0, 'machine': ('Linux', 'stkensrv3n.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 08:50:41 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'pstat': (33188, 47319392L, 20L, 1, 9811, 5063, 2118453701L, 1238357363, 1132850253, 1132849993), 'mode_octal': '0100744', 'mode': 33252, 'gname': 'root', 'size_bytes': 2118453701L, 'fullname': '/dev/null', 'inode': 47319392L, 'minor': 20}, 'version': 'v3_7d  CVS $Revision: 1.929 $ encp', 'encp_daq': None, 'client_crc': 1, 'infile': '/pnfs/cms/WAX/7/PHEDEX2/PCP04/Digi/eg03_jets_2g_pt50170.1/.(access)(000200000000000000D20960)', 'status': ('ok', None), 'volume': 'VOP775', 'outfile': '/dev/null', 'fc': {'status': ('ok', None), 'pnfsid': '000200000000000000D20960', 'pnfs_name0': '/pnfs/fs/usr/cms/WAX/7/PHEDEX2/PCP04/Digi/eg03_jets_2g_pt50170.1/ZippedEVD.63703329.63703340.eg_2x1033PU761_TkMu_2_g133_OSC.eg03_jets_2g_pt50170.zip', 'deleted': 'no', 'external_label': 'VOP775', 'drive': 'stkenmvr147a:/dev/rmt/tps0d0n:1310051195', 'update': '2009-02-26 18:19:54.189315', 'gid': 0, 'location_cookie': '0000_000000000_0000001', 'complete_crc': 4130209940L, 'address': ('131.225.13.59', 7501), 'uid': 0, 'bfid': 'CDMS123569399100000', 'sanity_cookie': (65536L, 1817439004L), 'size': 2118453701L}, 'infilepath': '/pnfs/cms/WAX/7/PHEDEX2/PCP04/Digi/eg03_jets_2g_pt50170.1/ZippedEVD.63703329.63703340.eg_2x1033PU761_TkMu_2_g133_OSC.eg03_jets_2g_pt50170.zip', 'at_the_top': 34, 'bfid': 'CDMS123569399100000', 'local_inode': 2469L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': 7, 'retry': 0}, 'callback_addr': ('131.225.13.51', 36441), 'work': 'read_from_hsm', 'override_ro_mount': 1, 'times': {'lm_dequeued': 1257400802.0088861, 'encp_start_time': 1257400799.204268, 'job_queued': 1257400800.313554, 't0': 1257400799, 'in_queue': 1.6952898502349854}, 'user_level': 'admin', 'unique_id': 'stkensrv3n.fnal.gov-1257400800-20124-0'} to mover
00:00:02 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:00:02 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV  READ FROM HSM Thread MainThread
00:00:02 stkensrv3n.fnal.gov 020124 root I ENCP  Listening for control socket at: ('131.225.13.51', 36441)
00:00:02 stkensrv3n.fnal.gov 020124 root I ENCP  Waiting for mover to connect control socket at address: ('131.225.13.51', 36441)  elapsed=2.830sec
00:00:02 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:00:02 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV  SENDING {'vc': {'comment': '<= VO5281 VO7669 VO9276 VO9363 VO9434 VO9484', 'declared': 1231186074.0, 'blocksize': 131072, 'storage_group': 'cms', 'library': 'CD-LTO4G1', 'eod_cookie': '0000_000000000_0001446', 'current_location': '0000_000000000_0000001', 'user_inhibit': ['none', 'none'], 'sum_rd_access': 1446, 'system_inhibit': ['none', 'full'], 'external_label': 'VOP775', 'wrapper': 'cpio_odc', 'remaining_bytes': 1525791744L, 'sum_mounts': 6, 'capacity_bytes': 858993459200L, 'media_type': 'LTO4', 'last_access': 1257400798.0, 'status': ('ok', None), 'si_time': [0.0, 1235726759.0], 'non_del_files': 1481, 'sum_wr_err': 0, 'sum_wr_access': 1445, 'file_family': 'cms7', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.cms7.cpio_odc', 'modification_time': 1238376412.0, 'write_protected': 'y', 'sum_rd_err': 0, 'first_access': 1235693914.0}, 'outfilepath': '/dev/null', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': 1, 'curpri': 1}, 'file_size': 2118453701L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'dismount_delay': 30, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7846, 'mc_device': '0,6,1,12', 'compression': 0, 'serial_num': '9311032672', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr146a', 'max_time_in_state': 1200, 'device': 'stkenmvr146a:/dev/rmt/tps0d0n', 'name': 'LTO4_46.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_46.stat', 'single_filemark': 1, 'mount_delay': 15, 'mover_address': ('131.225.13.174', 7846), 'check_first_written_file': 0, 'callback_addr': ('131.225.13.174', 44192), 'data_ip': 'stkenmvr146a', 'logname': 'LTO4_46MV', 'hostip': '131.225.13.174', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer', 'min_buffer': 67108864L}, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/cms/WAX/7/PHEDEX2/PCP04/Digi/eg03_jets_2g_pt50170.1/ZippedEVD.63703329.63703340.eg_2x1033PU761_TkMu_2_g133_OSC.eg03_jets_2g_pt50170.zip', 'uid': 0, 'uname': 'root', 'rmajor': 0, 'machine': ('Linux', 'stkensrv3n.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 08:50:41 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'pstat': (33188, 47319392L, 20L, 1, 9811, 5063, 2118453701L, 1238357363, 1132850253, 1132849993), 'mode_octal': '0100744', 'mode': 33252, 'size_bytes': 2118453701L, 'gname': 'root', 'gid': 0, 'fullname': '/dev/null', 'inode': 47319392L, 'minor': 20}, 'version': 'v3_7d  CVS $Revision: 1.929 $ encp', 'encp_daq': None, 'client_crc': 1, 'infile': '/pnfs/cms/WAX/7/PHEDEX2/PCP04/Digi/eg03_jets_2g_pt50170.1/.(access)(000200000000000000D20960)', 'status': ('ok', None), 'volume': 'VOP775', 'outfile': '/dev/null', 'fc': {'status': ('ok', None), 'pnfsid': '000200000000000000D20960', 'pnfs_name0': '/pnfs/fs/usr/cms/WAX/7/PHEDEX2/PCP04/Digi/eg03_jets_2g_pt50170.1/ZippedEVD.63703329.63703340.eg_2x1033PU761_TkMu_2_g133_OSC.eg03_jets_2g_pt50170.zip', 'deleted': 'no', 'external_label': 'VOP775', 'drive': 'stkenmvr147a:/dev/rmt/tps0d0n:1310051195', 'update': '2009-02-26 18:19:54.189315', 'gid': 0, 'location_cookie': '0000_000000000_0000001', 'complete_crc': 4130209940L, 'address': ('131.225.13.59', 7501), 'uid': 0, 'bfid': 'CDMS123569399100000', 'sanity_cookie': (65536L, 1817439004L), 'size': 2118453701L}, 'infilepath': '/pnfs/cms/WAX/7/PHEDEX2/PCP04/Digi/eg03_jets_2g_pt50170.1/ZippedEVD.63703329.63703340.eg_2x1033PU761_TkMu_2_g133_OSC.eg03_jets_2g_pt50170.zip', 'at_the_top': 34, 'bfid': 'CDMS123569399100000', 'local_inode': 2469L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': 7, 'retry': 0}, 'callback_addr': ('131.225.13.51', 36441), 'work': 'read_from_hsm', 'override_ro_mount': 1, 'times': {'lm_dequeued': 1257400802.0088861, 'encp_start_time': 1257400799.204268, 'job_queued': 1257400800.313554, 't0(trunc. 4205) Thread client_connect_thread
00:00:02 stkensrv3n.fnal.gov 020124 root I ENCP  Control socket ('131.225.13.51', 36441) is connected to ('131.225.13.174', 44193) for stkensrv3n.fnal.gov-1257400800-20124-0.  elapsed=2.850sec
00:00:02 stkensrv3n.fnal.gov 020124 root I ENCP  Received callback ticket from mover LTO4_46.mover for transfer stkensrv3n.fnal.gov-1257400800-20124-0.  elapsed=2.860sec
00:00:02 stkensrv3n.fnal.gov 020124 root I ENCP  Attempting to connect data socket to mover at ('131.225.13.174', 44192).
00:00:02 stkensrv3n.fnal.gov 020124 root I ENCP  Connecting data socket to mover (('131.225.13.174', 44192)) with interface 131.225.13.51.  elapsed=2.860sec
00:00:02 stkensrv3n.fnal.gov 020124 root I ENCP  Data socket ('131.225.13.51', 36442) is connected to ('131.225.13.174', 44192).  elapsed=2.860sec
00:00:02 stkensrv3n.fnal.gov 020124 root I ENCP  Opening local file.
00:00:02 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:00:02 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV  tape position: target 1 current 1288 54 Thread seek_thread
00:00:03 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:00:04 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:00:05 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:00:06 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:00:06 stkensrv0n.fnal.gov 023312 enstore I INFSRV  start listing all volumes (3)
00:00:07 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:00:07 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC  FINISHED listDrives returned ('ok', 0, None, '', '')
00:00:08 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:00:09 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:00:10 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:00:11 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:00:12 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:00:12 stkensrv4n.fnal.gov 005852 enstore I STKMC  FINISHED listDrives returned ('ok', 0, None, '', '')
00:00:13 stkendca17a.fnal.gov 030753 root I ENCP  Version: v3_8b  CVS $Revision: 1.922 $ <frozen>  OS: Linux 2.6.9-89.0.9.ELsmp i686 Release:  Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0)  Group: root(0)  Euser: root(0)  Egroup: root(0)  Library: CD-LTO4G1  Storage Group: exp-db  File Family: daily-d0-offline  FF Wrapper: cpio_odc  FF Width: 2  Current working directory: stkendca17a.fnal.gov:/opt/d-cache/config  Command line: /fnal/ups/prd/encp/v3_8b/Linux-2-6-dcache/encp --verbose 4 --threaded --ecrc --bypass-filesystem-max-filesize-check --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702070562_s103266_p22 --put-cache 0030000000000000061EB100 /diska/write-pool-1/data/0030000000000000061EB100
00:00:13 stkensrv4n.fnal.gov 005843 enstore I SL8500F1MC  FINISHED listDrives returned ('ok', 0, None, '', '')
00:00:13 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:00:14 stkendca17a.fnal.gov 030753 root I ENCP  Creating zero length output files.
00:00:14 stkendca17a.fnal.gov 030753 root I ENCP  Sending /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702070562_s103266_p22 write request to LM: unique_id: stkendca17a.fnal.gov-1257400813-30753-0 inputfile: /diska/write-pool-1/data/0030000000000000061EB100 outputfile: /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/.(access)(0030000000000000061EB100)
00:00:14 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:00:14 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  MSG_TYPE=ADD_TO_LMQ  request added to pending queue for storage group : exp-db
00:00:14 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  write Q'd /diska/write-pool-1/data/0030000000000000061EB100 (stkendca17a.fnal.gov-1257400813-30753-0) -> /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702070562_s103266_p22 : library=CD-LTO4G1 family=daily-d0-offline requester:root volume_family:exp-db.daily-d0-offline.cpio_odc
00:00:14 stkendca17a.fnal.gov 030753 root I ENCP  Listening for control socket at: ('131.225.13.5', 44347)
00:00:14 stkendca17a.fnal.gov 030753 root I ENCP  Waiting for mover to connect control socket at address: ('131.225.13.5', 44347)  elapsed=2.110sec
00:00:14 stkendca18a.fnal.gov 020159 root I ENCP  Version: v3_8b  CVS $Revision: 1.922 $ <frozen>  OS: Linux 2.6.9-89.0.9.ELsmp i686 Release:  Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0)  Group: root(0)  Euser: root(0)  Egroup: root(0)  Library: CD-LTO4G1  Storage Group: exp-db  File Family: daily-d0-offline  FF Wrapper: cpio_odc  FF Width: 2  Current working directory: stkendca18a.fnal.gov:/home/enstore/dcache-code/dcache-fermi-config  Command line: /fnal/ups/prd/encp/v3_8b/Linux-2-6-dcache/encp --verbose 4 --threaded --ecrc --bypass-filesystem-max-filesize-check --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702068213_s103263_p28 --put-cache 0030000000000000061EB0D8 /diskc/write-pool-3/data/0030000000000000061EB0D8
00:00:14 stkendca18a.fnal.gov 020159 root I ENCP  Creating zero length output files.
00:00:14 stkendca18a.fnal.gov 020159 root I ENCP  Sending /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702068213_s103263_p28 write request to LM: unique_id: stkendca18a.fnal.gov-1257400814-20159-0 inputfile: /diskc/write-pool-3/data/0030000000000000061EB0D8 outputfile: /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/.(access)(0030000000000000061EB0D8)
00:00:14 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  MSG_TYPE=ADD_TO_LMQ  request added to pending queue for storage group : exp-db
00:00:14 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  write Q'd /diskc/write-pool-3/data/0030000000000000061EB0D8 (stkendca18a.fnal.gov-1257400814-20159-0) -> /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702068213_s103263_p28 : library=CD-LTO4G1 family=daily-d0-offline requester:root volume_family:exp-db.daily-d0-offline.cpio_odc
00:00:14 stkendca18a.fnal.gov 020159 root I ENCP  Listening for control socket at: ('131.225.13.6', 54453)
00:00:14 stkendca18a.fnal.gov 020159 root I ENCP  Waiting for mover to connect control socket at address: ('131.225.13.6', 54453)  elapsed=0.980sec
00:00:15 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:00:16 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:00:16 stkensrv0n.fnal.gov 023312 enstore I INFSRV  stop listing all volumes (3)
00:00:17 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:00:18 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:00:19 last message repeated 1 times
00:00:19 stkendca17a.fnal.gov 030169 root I ENCP  File /diskb/write-pool-2/data/0030000000000000061EAFA0 transfered for stkendca17a.fnal.gov-1257400575-30169-0 in 20.8091299534 seconds.
00:00:19 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:00:20 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:00:21 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:00:21 stkendca22a.fnal.gov 015106 root I ENCP  Version: v3_8b  CVS $Revision: 1.922 $ <frozen>  OS: Linux 2.6.9-89.0.9.ELsmp i686 Release:  Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0)  Group: root(0)  Euser: root(0)  Egroup: root(0)  Library: garbage  Storage Group: test  File Family: dcache  FF Wrapper: cpio_odc  FF Width: 1  Current working directory: stkendca22a.fnal.gov:/home/enstore/dcache-log  Command line: /fnal/ups/prd/encp/v3_8b/Linux-2-6-dcache/encp --verbose 4 --threaded --ecrc --bypass-filesystem-max-filesize-check --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0003 --put-cache 0005000000000000005FA5F8 /diskb/write-pool-2/data/0005000000000000005FA5F8
00:00:21 stkendca22a.fnal.gov 015138 root I ENCP  Version: v3_8b  CVS $Revision: 1.922 $ <frozen>  OS: Linux 2.6.9-89.0.9.ELsmp i686 Release:  Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0)  Group: root(0)  Euser: root(0)  Egroup: root(0)  Library: garbage  Storage Group: test  File Family: dcache  FF Wrapper: cpio_odc  FF Width: 1  Current working directory: stkendca22a.fnal.gov:/home/enstore/dcache-log  Command line: /fnal/ups/prd/encp/v3_8b/Linux-2-6-dcache/encp --verbose 4 --threaded --ecrc --bypass-filesystem-max-filesize-check --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0001 --put-cache 0005000000000000005FA590 /diskb/write-pool-2/data/0005000000000000005FA590
00:00:21 stkendca22a.fnal.gov 015155 root I ENCP  Version: v3_8b  CVS $Revision: 1.922 $ <frozen>  OS: Linux 2.6.9-89.0.9.ELsmp i686 Release:  Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0)  Group: root(0)  Euser: root(0)  Egroup: root(0)  Library: garbage  Storage Group: test  File Family: dcache  FF Wrapper: cpio_odc  FF Width: 1  Current working directory: stkendca22a.fnal.gov:/home/enstore/dcache-log  Command line: /fnal/ups/prd/encp/v3_8b/Linux-2-6-dcache/encp --verbose 4 --threaded --ecrc --bypass-filesystem-max-filesize-check --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0002 --put-cache 0005000000000000005FA5C8 /diskb/write-pool-2/data/0005000000000000005FA5C8
00:00:22 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:00:22 stkendca22a.fnal.gov 015106 root I ENCP  Configuration Server: no such name: 'garbage.library_manager'
00:00:22 stkendca22a.fnal.gov 015106 root E ENCP  INFILE=/diskb/write-pool-2/data/0005000000000000005FA5F8 OUTFILE=/pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0003 FILESIZE=801512 LABEL= LOCATION= DRIVE= DRIVE_SN= TRANSFER_TIME=0.00 SEEK_TIME=0.00 MOUNT_TIME=0.00 QWAIT_TIME=0.00 TIME2NOW=0.99 CRC= STATUS=KEYERROR Configuration Server: no such name: 'garbage.library_manager'
00:00:22 stkendca22a.fnal.gov 015138 root I ENCP  Configuration Server: no such name: 'garbage.library_manager'
00:00:22 stkendca22a.fnal.gov 015138 root E ENCP  INFILE=/diskb/write-pool-2/data/0005000000000000005FA590 OUTFILE=/pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0001 FILESIZE=801512 LABEL= LOCATION= DRIVE= DRIVE_SN= TRANSFER_TIME=0.00 SEEK_TIME=0.00 MOUNT_TIME=0.00 QWAIT_TIME=0.00 TIME2NOW=1.01 CRC= STATUS=KEYERROR Configuration Server: no such name: 'garbage.library_manager'
00:00:22 stkendca22a.fnal.gov 015155 root I ENCP  Configuration Server: no such name: 'garbage.library_manager'
00:00:22 stkendca22a.fnal.gov 015155 root E ENCP  INFILE=/diskb/write-pool-2/data/0005000000000000005FA5C8 OUTFILE=/pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0002 FILESIZE=801512 LABEL= LOCATION= DRIVE= DRIVE_SN= TRANSFER_TIME=0.00 SEEK_TIME=0.00 MOUNT_TIME=0.00 QWAIT_TIME=0.00 TIME2NOW=1.01 CRC= STATUS=KEYERROR Configuration Server: no such name: 'garbage.library_manager'
00:00:23 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  written bytes 1966080512/1966080512, blocks 15001 header 188 trailer 324 Thread tape_thread
00:00:23 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:00:23 stkensrv4n.fnal.gov 005357 enstore E LTO4G1LM  MOVER ERROR RQ {'status': ('ok', None), 'returned_work': None, 'library': None, 'mover_type': '', 'current_location': '0000_000000000_0000000', 'ip_map': '', 'operation': None, 'read_only': 0, 'current_time': 1257400823.8541131, 'error_source': None, 'external_label': None, 'state': 'OFFLINE', 'r_a': (('131.225.13.194', 54880), 16278L, '131.225.13.194-54880-1256586929.509013-4866'), 'library_list': [('CD-LTO4G1.library_manager', ('131.225.13.4', 8541))], 'current_priority': (None, None), 'transfer_deficiency': 1, 'time_in_state': 813893.49405503273, 'address': ('131.225.13.194', 7860), 'volume_family': None, 'work': 'mover_error', 'mover': 'LTO4_99.mover', 'volume_status': (['none', 'none'], ['none', 'none']), 'unique_id': None}
00:00:24 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:00:24 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  filemarks written. Tape VON743 absolute location in blocks 1880476 Thread tape_thread
00:00:24 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  write_tape timing:pull 0.182555675507 write 11.7093307972 crc_check 7.82662129402 freespace 0.1980240345 block_write 19.9165318012 idle 0.316098928452 Thread tape_thread
00:00:24 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  drive stats after write. Tape VON743 position 129 block 0 block_size 0 bloc_loc 1880476 tot_blocks 762939000 BOT 0 read_err 0 write_err 0 bytes 1966080512 block_write_tot 19.9165318012 tape_rate 98716007.9688 Thread tape_thread
00:00:24 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  remaining bytes info in DB 547738624000 estimated 545772543488 reported from drive 545895424000 position 130 Thread tape_thread
00:00:24 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  new bitfile request {'uid': 0, 'external_label': 'VON743', 'gid': 0, 'location_cookie': '0000_000000000_0000129', 'complete_crc': 1022712303L, 'sanity_cookie': (65536L, 3024368633L), 'size': 1966080000L} Thread tape_thread
00:00:24 stkenmvr10a.fnal.gov 005910 root I DBT10MV  read bytes 9824152212/9824152212, blocks 74953 header 0 Thread tape_thread
00:00:24 stkenmvr10a.fnal.gov 005910 root I DBT10MV  drive stats after read. Tape VO6332 position 8 block 74953 block_size 0 bloc_loc 149720 tot_blocks 0 BOT 0 read_err 0 write_err 0 bytes 9824152212 block_read_tot 313.59986949 tape_rate 31327029.0195 Thread tape_thread
00:00:24 stkenmvr10a.fnal.gov 005910 root I DBT10MV  read_tape exiting, read 9824152212/9824152212 bytes Thread tape_thread
00:00:24 stkenmvr10a.fnal.gov 005910 root I DBT10MV  read_tape timing:get_space 1.19894266129 read 277.562609673 crc_check 33.5047652721 push 1.3335518837 block_read 313.59986949 idle 2.89731621742 Thread tape_thread
00:00:24 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  set remaining: VON743 545895424000 0000_000000000_0000130 Thread tape_thread
00:00:24 last message repeated 1 times
00:00:24 stkensrv0n.fnal.gov 028762 enstore I VOLSRV  Updating  			update volume set last_access = '2009-11-05 00:00:24', eod_cookie = '0000_000000000_0000130', non_del_files = '131', remaining_bytes = 545895424000 where label = 'VON743';
00:00:24 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  transfer complete volume=VON743 location=130 Thread tape_thread
00:00:24 last message repeated 1 times
00:00:24 stkensrv0n.fnal.gov 028762 enstore I VOLSRV  Updating  			update volume set sum_wr_access = '129', modification_time = '2009-11-05 00:00:24' where label = 'VON743';
00:00:24 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  Sending done to client: {'vc': {'comment': '', 'storage_group': 'exp-db', 'blocksize': 131072, 'declared': 1256853181.0, 'library': 'CD-LTO4G1', 'si_time': [0.0, 1255497368.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VON743', 'wrapper': 'cpio_odc', 'remaining_bytes': 547738624000L, 'sum_mounts': 46, 'sum_rd_access': 0, 'file_family_width': 2, 'last_access': 1257400798.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0000129', 'capacity_bytes': 858993459200L, 'non_del_files': 130, 'sum_wr_err': 0, 'sum_wr_access': 128, 'file_family': 'daily-d0-offline', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.daily-d0-offline.cpio_odc', 'modification_time': 1257400798.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1257340831.0}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702068213_s103263_p9', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 1966080000L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'dismount_delay': 30, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7842, 'mc_device': '0,7,1,12', 'compression': 0, 'serial_num': '1310155269', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr142a', 'max_time_in_state': 1200, 'device': 'stkenmvr142a:/dev/rmt/tps0d0n', 'name': 'LTO4_42.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_42.stat', 'single_filemark': 1, 'mount_delay': 15, 'mover_address': ('131.225.13.170', 7842), 'check_first_written_file': 0, 'callback_addr': ('131.225.13.170', 52216), 'data_ip': 'stkenmvr142a', 'logname': 'LTO4_42MV', 'hostip': '131.225.13.170', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer', 'min_buffer': 67108864L}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702068213_s103263_p9', 'uid': 0, 'uname': 'root', 'mtime': 1257400575, 'rmajor': 0, 'machine': ('Linux', 'stkendca17a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 820618800L, 20L, 1, 1602, 2752, 512L, 1257400556, 1257400556, 1254398418), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 1966080000L, 'gname': 'root', 'gid': 0, 'fullname': '/diskb/write-pool-2/data/0030000000000000061EAFA0', 'inode': 807317408L, 'minor': 20}, 'version': 'v3_8b  CVS $Revision: 1.922 $ <frozen>', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskb/write-pool-2/data/0030000000000000061EAFA0', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/.(access)(0030000000000000061EAFA0)', 'fc': {'uid': 0, 'external_label': 'VON743', 'gid': 0, 'location_cookie': '0000_000000000_0000129', 'complete_crc': 1022712303L, 'bfid': 'CDMS125740082400000', 'sanity_cookie': (65536L, 3024368633L), 'size': 1966080000L}, 'infilepath': '/diskb/write-pool-2/data/0030000000000000061EAFA0', 'at_the_top': 507, 'local_inode': 268435619L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.5', 44335), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257400574.7923689, 'seek_time': 0.033626794815063477, 'transfer_time': 26.150865077972412, 't0': 1257400574, 'mount_time': 0.039222002029418945, 'lm_dequeued': 1257400798.619575, 'job_queued': 1257400575.6963711, 'drive_call_time': 11.502760410308838, 'in_queue': 222.92291688919067, 'drive_transfer_time': 21.941827535629272}, 'user_level': 'u(trunc. 4142) Thread tape_thread
00:00:24 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  Done is sent Thread tape_thread
00:00:24 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  maybe_clean Thread tape_thread
00:00:24 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  returned from maybe_clean Thread tape_thread
00:00:24 stkendca17a.fnal.gov 030169 root I ENCP  Received final dialog for stkendca17a.fnal.gov-1257400575-30169-0.  elapsed=250.150sec
00:00:24 stkendca17a.fnal.gov 030169 root I ENCP  leaving transfer_file().
00:00:25 stkenmvr10a.fnal.gov 005910 root I DBT10MV  write_client: wrote 9824152212/9824152212 bytes Thread net_thread
00:00:25 stkenmvr10a.fnal.gov 005910 root I DBT10MV  write_client exits Thread net_thread
00:00:25 stkenmvr10a.fnal.gov 005910 root I DBT10MV  transfer complete volume=VO6332 location=8 Thread net_thread
00:00:25 last message repeated 1 times
00:00:25 stkensrv0n.fnal.gov 028762 enstore I VOLSRV  Updating  			update volume set sum_rd_access = '79', last_access = '2009-11-05 00:00:25' where label = 'VO6332';
00:00:25 stkenmvr10a.fnal.gov 005910 root I DBT10MV  Sending done to client: {'vc': {'comment': '', 'declared': 1083263100.0, 'blocksize': 131072, 'storage_group': 'ktev', 'library': 'CD-9940B', 'eod_cookie': '0000_000000000_0000679', 'current_location': '0000_000000000_0000008', 'user_inhibit': ['none', 'none'], 'sum_rd_access': 78, 'system_inhibit': ['none', 'migrating'], 'external_label': 'VO6332', 'wrapper': 'cern', 'remaining_bytes': 1683470848L, 'sum_mounts': 51, 'capacity_bytes': 214748364800L, 'media_type': '9940B', 'last_access': 1257400500.0, 'status': ('ok', None), 'si_time': [0.0, 1257400122.0], 'non_del_files': 226, 'sum_wr_err': 0, 'sum_wr_access': 226, 'file_family': 'ktev_migration', 'address': ('131.225.13.59', 7502), 'volume_family': 'ktev.ktev_migration.cern', 'modification_time': 1177350851.0, 'write_protected': 'y', 'sum_rd_err': 0, 'first_access': 1088397667.0}, 'outfilepath': '/data/data1/Migration_Spool/VO6332:0000_000000000_0000008', 'encp': {'agetime': 30, 'adminpri': 6, 'delpri': 1, 'basepri': 1000007, 'delayed_dismount': 2, 'curpri': 1000007}, 'file_size': 9824152212L, 'mover': {'max_buffer': 1048576000L, 'check_written_file': 10, 'max_failures': 5, 'library': 'CD-9940B.library_manager', 'local_mover': 0, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7575, 'mc_device': '0,0,10,15', 'compression': 0, 'serial_num': '479002012194', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 28311552.0, 'vendor_id': 'STK', 'driver': 'FTTDriver', 'host': 'stkenmvr10a', 'max_time_in_state': 1200, 'device': 'stkenmvr10a:/dev/rmt/tps2d0n', 'name': '9940B10.mover', 'send_stats': 1, 'product_id': 'T9940B', 'statistics_path': '/tmp/enstore/enstore/DBT10MV.stat', 'mover_address': ('131.225.13.37', 7575), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.37', 36790), 'data_ip': 'stkenmvr10a', 'logname': 'DBT10MV', 'hostip': '131.225.13.37', 'do_cleaning': 'No', 'media_changer': 'stk.media_changer'}, 'lm': {'address': ('131.225.13.4', 7522)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKE0/QKE078/f0', 'uid': 0, 'uname': 'root', 'rmajor': 0, 'machine': ('Linux', 'stkendm7a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (33188, 525010504L, 20L, 1, 5406, 2539, 1L, 1088398700, 1088398700, 1088354821), 'mode_octal': '0100744', 'mode': 33252, 'size_bytes': 9824152212L, 'gname': 'root', 'gid': 0, 'fullname': '/data/data1/Migration_Spool/VO6332:0000_000000000_0000008', 'inode': 525010504L, 'minor': 20}, 'version': 'v3_8  CVS $Revision: 1.963 $ encp.pyc', 'encp_daq': None, 'client_crc': 1, 'infile': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKE0/QKE078/.(access)(001F000000000000014B0648)', 'status': ('ok', None), 'volume': 'VO6332', 'outfile': '/data/data1/Migration_Spool/VO6332:0000_000000000_0000008', 'fc': {'status': ('ok', None), 'pnfsid': '001F000000000000014B0648', 'pnfs_name0': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKE0/QKE078/f0', 'deleted': 'no', 'external_label': 'VO6332', 'drive': 'stkenmvr23a:/dev/rmt/tps0d0n:4790000200', 'update': None, 'gid': 0, 'location_cookie': '0000_000000000_0000008', 'complete_crc': 89476965L, 'address': ('131.225.13.59', 7501), 'uid': 0, 'bfid': 'CDMS108839870000000', 'sanity_cookie': (65536L, 2108547209L), 'size': 9824152212L}, 'infilepath': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKE0/QKE078/f0', 'at_the_top': 979, 'bfid': 'CDMS108839870000000', 'local_inode': 39503169L, 'resend': {'max_retry': 3, 'resubmits': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.19', 50787), 'work': 'read_from_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257400123.7747331, 'seek_time': 0.0066330432891845703, 'transfer_time': 323.76157712936401, 't0': 1257400123, 'mount_time': 0.018432855606079102, 'lm_dequeued': 1257400501.1876941, 'job_queued': 1257400123.952822, 'drive_call_time': 275.43201684951782, 'in_queue': 377.23442697525024, 'drive_transfer_time': 315.09242296218872}, 'user_(trunc. 4150) Thread net_thread
00:00:25 stkenmvr10a.fnal.gov 005910 root I DBT10MV  Done is sent Thread net_thread
00:00:25 stkenmvr10a.fnal.gov 005910 root I DBT10MV  maybe_clean Thread net_thread
00:00:25 stkendca17a.fnal.gov 030169 root I ENCP  pf_status = ('ok', None)  skip_layer_cleanup = False
00:00:25 stkenmvr10a.fnal.gov 005910 root I DBT10MV  returned from maybe_clean Thread net_thread
00:00:25 stkensrv0n.fnal.gov 028796 enstore I FILSRV  Updating  			update file set deleted = 'n', pnfs_path = '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702068213_s103263_p9', drive = 'stkenmvr142a:/dev/rmt/tps0d0n:1310155269', pnfs_id = '0030000000000000061EAFA0' where bfid = 'CDMS125740082400000';
00:00:25 stkendca17a.fnal.gov 030169 root I ENCP  MSG_TYPE=ENCP_XFER    write_to_hsm /diskb/write-pool-2/data/0030000000000000061EAFA0 -> /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702068213_s103263_p9: 1966080000 bytes copied to VON743 at 71.6 MB/S (78.3 MB/S network) (85.5 MB/S drive) (90.1 MB/S disk) mover=LTO4_42.mover drive_id=ULTRIUM-TD4 drive_sn=1310155269 drive_vendor=IBM elapsed=250.49 {'storage_group': 'exp-db', 'encp_crc': 1022712303L, 'disk_rate': 90.12675571116435, 'drive_rate': 85.453228403849394, 'driver': 'FTTDriver', 'transfer_rate': 71.615185739184668, 'mover_interface': 'stkenmvr142a', 'overall_rate': 70.466625540705635, 'media_changer': 'SL8500G1.media_changer', 'network_rate': 71.615185739184668, 'encp_ip': '131.225.13.5', 'unique_id': 'stkendca17a.fnal.gov-1257400575-30169-0'}
00:00:25 stkendca17a.fnal.gov 030169 root I ENCP  File /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702068213_s103263_p9 copied successfully.
00:00:25 stkendca17a.fnal.gov 030169 root I ENCP  Completed transferring 1966080000 bytes in 1 files in 250.490352154 sec.  Overall rate = 70.5 MB/sec.  Transfer rate = 71.6 MB/sec.  Network rate = 78.3 MB/sec.  Drive rate = 85.5 MB/sec.  Disk rate = 90.1 MB/sec.  Exit status = 0.
00:00:25 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  <storage groups {'ktev': [('9940B35.mover', 'VO6341'), ('9940B10.mover', 'VO6332')]} volume_families {'ktev.ktev_migration.cern': [('9940B35.mover', 'VO6341'), ('9940B10.mover', 'VO6332')]} >
00:00:25 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  read_from_hsm next work on vol=VO6332 mover=9940B10.mover requester:root
00:00:25 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  HAVE_BOUND:sending read_from_hsm /pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKE0/QKE090/f0 to mover 9940B10.mover ('131.225.13.37', 7575) DEL_DISM 2
00:00:25 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  MSG_TYPE=ADD_TO_LMQ  request added to work at movers queue for storage group : ktev
00:00:25 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  HAVE_BOUND:sending {'vc': {'comment': '', 'declared': 1083263100.0, 'blocksize': 131072, 'storage_group': 'ktev', 'library': 'CD-9940B', 'eod_cookie': '0000_000000000_0000679', 'current_location': '0000_000000000_0000011', 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'migrating'], 'external_label': 'VO6332', 'wrapper': 'cern', 'remaining_bytes': 1683470848L, 'sum_mounts': 51, 'sum_rd_access': 77, 'media_type': '9940B', 'last_access': 1257400490.0, 'status': ('ok', None), 'si_time': [0.0, 1257400122.0], 'capacity_bytes': 214748364800L, 'non_del_files': 226, 'sum_wr_err': 0, 'sum_wr_access': 226, 'file_family': 'ktev_migration', 'address': ('131.225.13.59', 7502), 'volume_family': 'ktev.ktev_migration.cern', 'modification_time': 1177350851.0, 'write_protected': 'y', 'sum_rd_err': 0, 'first_access': 1088397667.0}, 'outfilepath': '/data/data1/Migration_Spool/VO6332:0000_000000000_0000011', 'encp': {'agetime': 30, 'adminpri': 6, 'delpri': 1, 'basepri': 1000007, 'delayed_dismount': 2, 'curpri': 1000007}, 'file_size': 1891545456L, 'mover': '9940B10.mover', 'lm': {'address': ('131.225.13.4', 7522)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKE0/QKE090/f0', 'uid': 0, 'uname': 'root', 'gid': 0, 'rmajor': 0, 'machine': ('Linux', 'stkendm7a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (33188, 525017360L, 20L, 1, 5406, 2539, 1891545456L, 1088398805, 1088398805, 1088367178), 'mode_octal': '0100744', 'mode': 33252, 'gname': 'root', 'size_bytes': 1891545456L, 'fullname': '/data/data1/Migration_Spool/VO6332:0000_000000000_0000011', 'inode': 525017360L, 'minor': 20}, 'version': 'v3_8  CVS $Revision: 1.963 $ encp.pyc', 'encp_daq': None, 'client_crc': 1, 'infile': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKE0/QKE090/.(access)(001F000000000000014B2110)', 'status': ('ok', None), 'volume': 'VO6332', 'outfile': '/data/data1/Migration_Spool/VO6332:0000_000000000_0000011', 'fc': {'status': ('ok', None), 'pnfsid': '001F000000000000014B2110', 'pnfs_name0': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKE0/QKE090/f0', 'deleted': 'no', 'external_label': 'VO6332', 'drive': 'stkenmvr23a:/dev/rmt/tps0d0n:4790000200', 'update': None, 'gid': 0, 'location_cookie': '0000_000000000_0000011', 'complete_crc': 2899242434L, 'address': ('131.225.13.59', 7501), 'uid': 0, 'bfid': 'CDMS108839880300000', 'sanity_cookie': (65536L, 115691675L), 'size': 1891545456L}, 'infilepath': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKE0/QKE090/f0', 'at_the_top': 1792, 'bfid': 'CDMS108839880300000', 'local_inode': 39503171L, 'resend': {'max_retry': 3, 'resubmits': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.19', 50802), 'work': 'read_from_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257400825.3903811, 'encp_start_time': 1257400491.869041, 'job_queued': 1257400492.035696, 't0': 1257400491, 'in_queue': 333.35422801971436}, 'user_level': 'admin', 'unique_id': 'stkendm7a.fnal.gov-1257400491-31675-3'} to mover
00:00:25 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  <storage groups {'ktev': [('9940B35.mover', 'VO6341'), ('9940B10.mover', 'VO6332')]} volume_families {'ktev.ktev_migration.cern': [('9940B35.mover', 'VO6341'), ('9940B10.mover', 'VO6332')]} >
00:00:25 stkenmvr10a.fnal.gov 005910 root I DBT10MV  READ FROM HSM Thread MainThread
00:00:25 stkenmvr10a.fnal.gov 005910 root I DBT10MV  SENDING {'vc': {'comment': '', 'declared': 1083263100.0, 'blocksize': 131072, 'storage_group': 'ktev', 'library': 'CD-9940B', 'eod_cookie': '0000_000000000_0000679', 'current_location': '0000_000000000_0000011', 'user_inhibit': ['none', 'none'], 'sum_rd_access': 77, 'system_inhibit': ['none', 'migrating'], 'external_label': 'VO6332', 'wrapper': 'cern', 'remaining_bytes': 1683470848L, 'sum_mounts': 51, 'capacity_bytes': 214748364800L, 'media_type': '9940B', 'last_access': 1257400490.0, 'status': ('ok', None), 'si_time': [0.0, 1257400122.0], 'non_del_files': 226, 'sum_wr_err': 0, 'sum_wr_access': 226, 'file_family': 'ktev_migration', 'address': ('131.225.13.59', 7502), 'volume_family': 'ktev.ktev_migration.cern', 'modification_time': 1177350851.0, 'write_protected': 'y', 'sum_rd_err': 0, 'first_access': 1088397667.0}, 'outfilepath': '/data/data1/Migration_Spool/VO6332:0000_000000000_0000011', 'encp': {'agetime': 30, 'adminpri': 6, 'delpri': 1, 'basepri': 1000007, 'delayed_dismount': 2, 'curpri': 1000007}, 'file_size': 1891545456L, 'mover': {'max_buffer': 1048576000L, 'check_written_file': 10, 'max_failures': 5, 'library': 'CD-9940B.library_manager', 'local_mover': 0, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7575, 'mc_device': '0,0,10,15', 'compression': 0, 'serial_num': '479002012194', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 28311552.0, 'vendor_id': 'STK', 'driver': 'FTTDriver', 'host': 'stkenmvr10a', 'max_time_in_state': 1200, 'device': 'stkenmvr10a:/dev/rmt/tps2d0n', 'name': '9940B10.mover', 'send_stats': 1, 'product_id': 'T9940B', 'statistics_path': '/tmp/enstore/enstore/DBT10MV.stat', 'mover_address': ('131.225.13.37', 7575), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.37', 36806), 'data_ip': 'stkenmvr10a', 'logname': 'DBT10MV', 'hostip': '131.225.13.37', 'do_cleaning': 'No', 'media_changer': 'stk.media_changer'}, 'lm': {'address': ('131.225.13.4', 7522)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKE0/QKE090/f0', 'uid': 0, 'uname': 'root', 'rmajor': 0, 'machine': ('Linux', 'stkendm7a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (33188, 525017360L, 20L, 1, 5406, 2539, 1891545456L, 1088398805, 1088398805, 1088367178), 'mode_octal': '0100744', 'mode': 33252, 'size_bytes': 1891545456L, 'gname': 'root', 'gid': 0, 'fullname': '/data/data1/Migration_Spool/VO6332:0000_000000000_0000011', 'inode': 525017360L, 'minor': 20}, 'version': 'v3_8  CVS $Revision: 1.963 $ encp.pyc', 'encp_daq': None, 'client_crc': 1, 'infile': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKE0/QKE090/.(access)(001F000000000000014B2110)', 'status': ('ok', None), 'volume': 'VO6332', 'outfile': '/data/data1/Migration_Spool/VO6332:0000_000000000_0000011', 'fc': {'status': ('ok', None), 'pnfsid': '001F000000000000014B2110', 'pnfs_name0': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKE0/QKE090/f0', 'deleted': 'no', 'external_label': 'VO6332', 'drive': 'stkenmvr23a:/dev/rmt/tps0d0n:4790000200', 'update': None, 'gid': 0, 'location_cookie': '0000_000000000_0000011', 'complete_crc': 2899242434L, 'address': ('131.225.13.59', 7501), 'uid': 0, 'bfid': 'CDMS108839880300000', 'sanity_cookie': (65536L, 115691675L), 'size': 1891545456L}, 'infilepath': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKE0/QKE090/f0', 'at_the_top': 1792, 'bfid': 'CDMS108839880300000', 'local_inode': 39503171L, 'resend': {'max_retry': 3, 'resubmits': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.19', 50802), 'work': 'read_from_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257400825.3903811, 'encp_start_time': 1257400491.869041, 'job_queued': 1257400492.035696, 't0': 1257400491, 'in_queue': 333.35422801971436}, 'user_level': 'admin', 'unique_id': 'stkendm7a.fnal.gov-1257400491-31675-3'} Thread client_connect_thread
00:00:25 stkendm7a.fnal.gov 031675 root I ENCP  Listening for control socket at: ('131.225.13.19', 50802) Thread READ
00:00:25 stkendm7a.fnal.gov 031675 root I ENCP  Waiting for mover to connect control socket at address: ('131.225.13.19', 50802)  elapsed=702.510sec Thread READ
00:00:25 stkendm7a.fnal.gov 031675 root I ENCP  Control socket ('131.225.13.19', 50802) is connected to ('131.225.13.37', 36807) for stkendm7a.fnal.gov-1257400491-31675-3.  elapsed=702.520sec Thread READ
00:00:25 stkendm7a.fnal.gov 031675 root I ENCP  Received callback ticket from mover 9940B10.mover for transfer stkendm7a.fnal.gov-1257400491-31675-3.  elapsed=702.520sec Thread READ
00:00:25 stkendm7a.fnal.gov 031675 root I ENCP  Attempting to connect data socket to mover at ('131.225.13.37', 36806). Thread READ
00:00:25 stkendm7a.fnal.gov 031675 root I ENCP  Connecting data socket to mover (('131.225.13.37', 36806)) with interface 131.225.13.19.  elapsed=702.520sec Thread READ
00:00:25 stkendm7a.fnal.gov 031675 root I ENCP  Data socket ('131.225.13.19', 50816) is connected to ('131.225.13.37', 36806).  elapsed=702.520sec Thread READ
00:00:25 stkendm7a.fnal.gov 031675 root I ENCP  Opening local file. Thread READ
00:00:25 stkenmvr10a.fnal.gov 005910 root I DBT10MV  tape position: target 11 current 8 74953 Thread seek_thread
00:00:25 stkenmvr10a.fnal.gov 005910 root I DBT10MV  skip_fm 3 Thread seek_thread
00:00:25 stkenmvr10a.fnal.gov 005910 root I DBT10MV  read_tape starting, bytes_to_read=1891545456 Thread tape_thread
00:00:25 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:00:25 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  write_to_hsm next work on vol=VON743 mover=LTO4_42.mover requester:root
00:00:25 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  HAVE_BOUND:sending write_to_hsm /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702065710_s103260_p1 to mover LTO4_42.mover ('131.225.13.170', 7842) DEL_DISM None
00:00:25 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  MSG_TYPE=ADD_TO_LMQ  request added to work at movers queue for storage group : exp-db
00:00:25 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  HAVE_BOUND:sending {'vc': {'storage_group': 'exp-db', 'external_label': 'VON743', 'library': 'CD-LTO4G1', 'file_family': 'daily-d0-offline', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.daily-d0-offline.cpio_odc', 'file_family_width': 2}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702065710_s103260_p1', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 1966080000L, 'mover': 'LTO4_42.mover', 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702065710_s103260_p1', 'uid': 0, 'uname': 'root', 'type': 'cpio_odc', 'gid': 0, 'mtime': 1257400742, 'rmajor': 0, 'machine': ('Linux', 'stkendca19a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (16877, 820618800L, 20L, 1, 1602, 2752, 512L, 1257400711, 1257400711, 1254398418), 'mode_octal': '0100016', 'mode': 33188, 'gname': 'root', 'size_bytes': 1966080000L, 'fullname': '/diska/write-pool-1/data/0030000000000000061EB070', 'inode': 807317616L, 'minor': 20}, 'version': 'v3_8b  CVS $Revision: 1.922 $ <frozen>', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diska/write-pool-1/data/0030000000000000061EB070', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/.(access)(0030000000000000061EB070)', 'fc': {'size': 1966080000L, 'external_label': 'VON743', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/diska/write-pool-1/data/0030000000000000061EB070', 'at_the_top': 379, 'local_inode': 628421102L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.11', 47715), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257400825.655771, 'encp_start_time': 1257400741.3442709, 'job_queued': 1257400742.235882, 't0': 1257400741, 'in_queue': 83.419617891311646}, 'user_level': 'user2', 'unique_id': 'stkendca19a.fnal.gov-1257400742-18418-0'} to mover
00:00:25 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:00:25 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  WRITE_TO_HSM Thread MainThread
00:00:25 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  SENDING {'vc': {'storage_group': 'exp-db', 'external_label': 'VON743', 'library': 'CD-LTO4G1', 'file_family': 'daily-d0-offline', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.daily-d0-offline.cpio_odc', 'file_family_width': 2}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702065710_s103260_p1', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 1966080000L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'dismount_delay': 30, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7842, 'mc_device': '0,7,1,12', 'compression': 0, 'serial_num': '1310155269', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr142a', 'max_time_in_state': 1200, 'device': 'stkenmvr142a:/dev/rmt/tps0d0n', 'name': 'LTO4_42.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_42.stat', 'single_filemark': 1, 'mount_delay': 15, 'mover_address': ('131.225.13.170', 7842), 'check_first_written_file': 0, 'callback_addr': ('131.225.13.170', 52222), 'data_ip': 'stkenmvr142a', 'logname': 'LTO4_42MV', 'hostip': '131.225.13.170', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer', 'min_buffer': 67108864L}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702065710_s103260_p1', 'uid': 0, 'uname': 'root', 'mtime': 1257400742, 'rmajor': 0, 'machine': ('Linux', 'stkendca19a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 820618800L, 20L, 1, 1602, 2752, 512L, 1257400711, 1257400711, 1254398418), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 1966080000L, 'gname': 'root', 'gid': 0, 'fullname': '/diska/write-pool-1/data/0030000000000000061EB070', 'inode': 807317616L, 'minor': 20}, 'version': 'v3_8b  CVS $Revision: 1.922 $ <frozen>', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diska/write-pool-1/data/0030000000000000061EB070', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/.(access)(0030000000000000061EB070)', 'fc': {'size': 1966080000L, 'external_label': 'VON743', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/diska/write-pool-1/data/0030000000000000061EB070', 'at_the_top': 379, 'local_inode': 628421102L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.11', 47715), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257400825.655771, 'encp_start_time': 1257400741.3442709, 'job_queued': 1257400742.235882, 't0': 1257400741, 'in_queue': 83.419617891311646}, 'user_level': 'user2', 'unique_id': 'stkendca19a.fnal.gov-1257400742-18418-0'} Thread client_connect_thread
00:00:25 stkendca19a.fnal.gov 018418 root I ENCP  Control socket ('131.225.13.11', 47715) is connected to ('131.225.13.170', 52223) for stkendca19a.fnal.gov-1257400742-18418-0.  elapsed=84.380sec
00:00:25 stkendca19a.fnal.gov 018418 root I ENCP  Received callback ticket from mover LTO4_42.mover for transfer stkendca19a.fnal.gov-1257400742-18418-0.  elapsed=84.380sec
00:00:25 stkendca19a.fnal.gov 018418 root I ENCP  Atempting to connect data socket to mover at ('131.225.13.170', 52222).
00:00:25 stkendca19a.fnal.gov 018418 root I ENCP  Connecting data socket to mover (('131.225.13.170', 52222)) with interface 131.225.13.11.  elapsed=84.380sec
00:00:25 stkendca19a.fnal.gov 018418 root I ENCP  Data socket ('131.225.13.11', 47719) is connected to ('131.225.13.170', 52222).  elapsed=84.380sec
00:00:25 stkendca19a.fnal.gov 018418 root I ENCP  Opening local file.
00:00:25 stkendca19a.fnal.gov 018418 root I ENCP  pf_status = ('ok', None)  skip_layer_cleanup = False
00:00:25 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  tape position: target 130 current 130 0 Thread seek_thread
00:00:25 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  write_tape starting, bytes_to_write=1966080512 Thread tape_thread
00:00:25 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:00:25 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  Write starting. Tape VON743 absolute location in blocks 1880476 Thread tape_thread
00:00:25 stkendm7a.fnal.gov 031675 root I ENCP  File /pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKE0/QKE078/f0 transfered for stkendm7a.fnal.gov-1257400123-31675-1 in 324.556020021 seconds. Thread READ_2
00:00:25 stkendm7a.fnal.gov 031675 root I ENCP  Received final dialog for stkendm7a.fnal.gov-1257400123-31675-1.  elapsed=702.920sec Thread READ_2
00:00:25 stkendm7a.fnal.gov 031675 root I ENCP  leaving transfer_file(). Thread READ_2
00:00:26 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:00:26 stkendm7a.fnal.gov 031675 root I ENCP  MSG_TYPE=ENCP_XFER    read_from_hsm /pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKE0/QKE078/f0 -> /data/data1/Migration_Spool/VO6332:0000_000000000_0000008: 9824152212 bytes copied from VO6332 at 28.9 MB/S (37 MB/S network) (29.7 MB/S drive) (67.4 MB/S disk) mover=9940B10.mover drive_id=T9940B drive_sn=479002012194 drive_vendor=STK elapsed=702.35 {'storage_group': 'ktev', 'encp_crc': 89476965L, 'disk_rate': 67.377193944983262, 'drive_rate': 29.734265130756061, 'driver': 'FTTDriver', 'transfer_rate': 28.866084437401106, 'mover_interface': 'stkenmvr10a', 'overall_rate': 28.838932293559939, 'media_changer': 'stk.media_changer', 'network_rate': 28.866084437401106, 'encp_ip': '131.225.13.19', 'unique_id': 'stkendm7a.fnal.gov-1257400123-31675-1'} Thread READ_2
00:00:26 stkendm7a.fnal.gov 031675 root I ENCP  File /data/data1/Migration_Spool/VO6332:0000_000000000_0000008 copied successfully. Thread READ_2
00:00:26 stkendm7a.fnal.gov 031675 root I ENCP  Completed transferring 9824152212 bytes in 1 files in 702.369855881 sec.  Overall rate = 28.8 MB/sec.  Transfer rate = 28.9 MB/sec.  Network rate = 37 MB/sec.  Drive rate = 29.7 MB/sec.  Disk rate = 67.4 MB/sec.  Exit status = 0. Thread READ_2
00:00:26 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  <storage groups {'ktev': [('9940B35.mover', 'VO6341'), ('9940B10.mover', 'VO6332')]} volume_families {'ktev.ktev_migration.cern': [('9940B35.mover', 'VO6341'), ('9940B10.mover', 'VO6332')]} >
00:00:26 stkendm7a.fnal.gov 031675 root I ENCP  Version: v3_8  CVS $Revision: 1.963 $ encp.pyc  OS: Linux 2.6.9-89.0.9.ELsmp i686 Release:  Scientific Linux Fermi LTS release 4.4 (Wilson), User: root(0)  Group: root(0)  Euser: root(0)  Egroup: root(0)  Current working directory: stkendm7a.fnal.gov:/opt/enstore/src  Command line: encp --delayed-dismount 2 --ignore-fair-share --bypass-filesystem-max-filesize-check --priority 0 --threaded /pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKCC/QKCC16/f2 /data/data1/Migration_Spool/VO6332:0000_000000000_0000017 Thread READ_2
00:00:26 stkendm7a.fnal.gov 031675 root I ENCP  Creating zero length output files. Thread READ_2
00:00:26 stkendm7a.fnal.gov 031675 root I ENCP  Sending /pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKCC/QKCC16/f2 read request to LM: unique_id: stkendm7a.fnal.gov-1257400826-31675-5 inputfile: /pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKCC/QKCC16/.(access)(001F000000000000014B0D80) outputfile: /data/data1/Migration_Spool/VO6332:0000_000000000_0000017 Thread READ_2
00:00:26 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  MSG_TYPE=ADD_TO_LMQ  request added to pending queue for storage group : ktev
00:00:26 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  read Q'd /pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKCC/QKCC16/f2 (stkendm7a.fnal.gov-1257400826-31675-5) -> /data/data1/Migration_Spool/VO6332:0000_000000000_0000017 : library=CD-9940B family=ktev.ktev_migration.cern requester:root
00:00:26 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:00:26 stkendca21a.fnal.gov 000378 root I ENCP  Version: v3_8b  CVS $Revision: 1.922 $ <frozen>  OS: Linux 2.6.9-89.0.9.ELsmp i686 Release:  Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0)  Group: root(0)  Euser: root(0)  Egroup: root(0)  Library: CD-LTO4G1  Storage Group: minos  File Family: minos  FF Wrapper: cpio_odc  FF Width: 1  Current working directory: stkendca21a.fnal.gov:/  Command line: /fnal/ups/prd/encp/v3_8b/Linux-2-6-dcache/encp --verbose 4 --threaded --ecrc --bypass-filesystem-max-filesize-check --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/711/n13047112_0023_L010185R_D07_r4.cand.dogwood1.0.root --put-cache 000F0000000000000B2471F0 /diskb/write-pool-2/data/000F0000000000000B2471F0
00:00:26 stkendca21a.fnal.gov 000378 root I ENCP  Creating zero length output files.
00:00:26 stkendca21a.fnal.gov 000378 root I ENCP  Sending /pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/711/n13047112_0023_L010185R_D07_r4.cand.dogwood1.0.root write request to LM: unique_id: stkendca21a.fnal.gov-1257400826-378-0 inputfile: /diskb/write-pool-2/data/000F0000000000000B2471F0 outputfile: /pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/711/.(access)(000F0000000000000B2471F0)
00:00:26 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  MSG_TYPE=ADD_TO_LMQ  request added to pending queue for storage group : minos
00:00:26 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  write Q'd /diskb/write-pool-2/data/000F0000000000000B2471F0 (stkendca21a.fnal.gov-1257400826-378-0) -> /pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/711/n13047112_0023_L010185R_D07_r4.cand.dogwood1.0.root : library=CD-LTO4G1 family=minos requester:root volume_family:minos.minos.cpio_odc
00:00:26 stkendca21a.fnal.gov 000378 root I ENCP  Listening for control socket at: ('131.225.13.21', 44456)
00:00:26 stkendca21a.fnal.gov 000378 root I ENCP  Waiting for mover to connect control socket at address: ('131.225.13.21', 44456)  elapsed=0.870sec
00:00:27 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:00:28 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:00:28 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  write_to_hsm next work on vol=VON262 mover=LTO4_109.mover requester:root
00:00:28 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  HAVE_BOUND:sending write_to_hsm /pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/711/n13047112_0023_L010185R_D07_r4.cand.dogwood1.0.root to mover LTO4_109.mover ('131.225.13.205', 7869) DEL_DISM None
00:00:28 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  MSG_TYPE=ADD_TO_LMQ  request added to work at movers queue for storage group : minos
00:00:28 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  HAVE_BOUND:sending {'vc': {'storage_group': 'minos', 'external_label': 'VON262', 'library': 'CD-LTO4G1', 'file_family': 'minos', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'minos.minos.cpio_odc', 'file_family_width': 1}, 'outfilepath': '/pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/711/n13047112_0023_L010185R_D07_r4.cand.dogwood1.0.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 387537400L, 'mover': 'LTO4_109.mover', 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/711/n13047112_0023_L010185R_D07_r4.cand.dogwood1.0.root', 'uid': 0, 'uname': 'root', 'type': 'cpio_odc', 'gid': 0, 'mtime': 1257400826, 'rmajor': 0, 'machine': ('Linux', 'stkendca21a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (16893, 253994256L, 20L, 1, 42411, 5111, 512L, 1257400777, 1257400777, 1257381850), 'mode_octal': '0100036', 'mode': 33188, 'gname': 'root', 'size_bytes': 387537400L, 'fullname': '/diskb/write-pool-2/data/000F0000000000000B2471F0', 'inode': 254046704L, 'minor': 20}, 'version': 'v3_8b  CVS $Revision: 1.922 $ <frozen>', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskb/write-pool-2/data/000F0000000000000B2471F0', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/711/.(access)(000F0000000000000B2471F0)', 'fc': {'size': 387537400L, 'external_label': 'VON262', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/diskb/write-pool-2/data/000F0000000000000B2471F0', 'at_the_top': 9, 'local_inode': 1073743790L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.21', 44456), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257400828.1960149, 'encp_start_time': 1257400826.097872, 'job_queued': 1257400826.948622, 't0': 1257400826, 'in_queue': 1.2473349571228027}, 'user_level': 'user2', 'unique_id': 'stkendca21a.fnal.gov-1257400826-378-0'} to mover
00:00:28 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:00:28 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  WRITE_TO_HSM Thread MainThread
00:00:28 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  SENDING {'vc': {'storage_group': 'minos', 'external_label': 'VON262', 'library': 'CD-LTO4G1', 'file_family': 'minos', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'minos.minos.cpio_odc', 'file_family_width': 1}, 'outfilepath': '/pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/711/n13047112_0023_L010185R_D07_r4.cand.dogwood1.0.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 387537400L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7869, 'mc_device': '0,5,1,7', 'compression': 0, 'serial_num': '9310025229', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr209a', 'max_time_in_state': 1200, 'device': 'stkenmvr209a:/dev/rmt/tps0d0n', 'name': 'LTO4_109.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_109.stat', 'mover_address': ('131.225.13.205', 7869), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.205', 42015), 'data_ip': 'stkenmvr209a', 'logname': 'LTO4_109MV', 'hostip': '131.225.13.205', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/711/n13047112_0023_L010185R_D07_r4.cand.dogwood1.0.root', 'uid': 0, 'uname': 'root', 'mtime': 1257400826, 'rmajor': 0, 'machine': ('Linux', 'stkendca21a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16893, 253994256L, 20L, 1, 42411, 5111, 512L, 1257400777, 1257400777, 1257381850), 'mode_octal': '0100036', 'mode': 33188, 'size_bytes': 387537400L, 'gname': 'root', 'gid': 0, 'fullname': '/diskb/write-pool-2/data/000F0000000000000B2471F0', 'inode': 254046704L, 'minor': 20}, 'version': 'v3_8b  CVS $Revision: 1.922 $ <frozen>', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskb/write-pool-2/data/000F0000000000000B2471F0', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/711/.(access)(000F0000000000000B2471F0)', 'fc': {'size': 387537400L, 'external_label': 'VON262', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/diskb/write-pool-2/data/000F0000000000000B2471F0', 'at_the_top': 9, 'local_inode': 1073743790L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.21', 44456), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257400828.1960149, 'encp_start_time': 1257400826.097872, 'job_queued': 1257400826.948622, 't0': 1257400826, 'in_queue': 1.2473349571228027}, 'user_level': 'user2', 'unique_id': 'stkendca21a.fnal.gov-1257400826-378-0'} Thread client_connect_thread
00:00:28 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:00:28 stkendca21a.fnal.gov 000378 root I ENCP  Control socket ('131.225.13.21', 44456) is connected to ('131.225.13.205', 42016) for stkendca21a.fnal.gov-1257400826-378-0.  elapsed=2.130sec
00:00:28 stkendca21a.fnal.gov 000378 root I ENCP  Received callback ticket from mover LTO4_109.mover for transfer stkendca21a.fnal.gov-1257400826-378-0.  elapsed=2.130sec
00:00:28 stkendca21a.fnal.gov 000378 root I ENCP  Atempting to connect data socket to mover at ('131.225.13.205', 42015).
00:00:28 stkendca21a.fnal.gov 000378 root I ENCP  Connecting data socket to mover (('131.225.13.205', 42015)) with interface 131.225.13.21.  elapsed=2.130sec
00:00:28 stkendca21a.fnal.gov 000378 root I ENCP  Data socket ('131.225.13.21', 44457) is connected to ('131.225.13.205', 42015).  elapsed=2.140sec
00:00:28 stkendca21a.fnal.gov 000378 root I ENCP  Opening local file.
00:00:28 stkendca21a.fnal.gov 000378 root I ENCP  pf_status = ('ok', None)  skip_layer_cleanup = False
00:00:28 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  tape position: target 856 current 856 0 Thread seek_thread
00:00:28 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  write_tape starting, bytes_to_write=387537920 Thread tape_thread
00:00:28 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  Write starting. Tape VON262 absolute location in blocks 2531652 Thread tape_thread
00:00:29 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:00:32 last message repeated 1 times
00:00:32 stkendca21a.fnal.gov 000378 root I ENCP  File /diskb/write-pool-2/data/000F0000000000000B2471F0 transfered for stkendca21a.fnal.gov-1257400826-378-0 in 4.53129601479 seconds.
00:00:32 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  written bytes 387537920/387537920, blocks 2957 header 213 trailer 307 Thread tape_thread
00:00:33 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  filemarks written. Tape VON262 absolute location in blocks 2534610 Thread tape_thread
00:00:33 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  write_tape timing:pull 0.0327739715576 write 0.356701374054 crc_check 1.81763315201 freespace 0.038498878479 block_write 2.2456073761 idle 0.0615484714508 Thread tape_thread
00:00:33 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  drive stats after write. Tape VON262 position 856 block 0 block_size 0 bloc_loc 2534610 tot_blocks 762939000 BOT 0 read_err 0 write_err 0 bytes 387537920 block_write_tot 2.2456073761 tape_rate 172575991.745 Thread tape_thread
00:00:33 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  remaining bytes info in DB 465168384000 estimated 464780846080 reported from drive 464807936000 position 857 Thread tape_thread
00:00:33 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  new bitfile request {'uid': 0, 'external_label': 'VON262', 'gid': 0, 'location_cookie': '0000_000000000_0000856', 'complete_crc': 3922966897L, 'sanity_cookie': (65536L, 3368846051L), 'size': 387537400L} Thread tape_thread
00:00:33 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  set remaining: VON262 464807936000 0000_000000000_0000857 Thread tape_thread
00:00:33 last message repeated 1 times
00:00:33 stkensrv0n.fnal.gov 028762 enstore I VOLSRV  Updating  			update volume set last_access = '2009-11-05 00:00:33', eod_cookie = '0000_000000000_0000857', non_del_files = '859', remaining_bytes = 464807936000 where label = 'VON262';
00:00:33 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  transfer complete volume=VON262 location=857 Thread tape_thread
00:00:33 last message repeated 1 times
00:00:33 stkensrv0n.fnal.gov 028762 enstore I VOLSRV  Updating  			update volume set sum_wr_access = '856', modification_time = '2009-11-05 00:00:33' where label = 'VON262';
00:00:33 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  Sending done to client: {'vc': {'comment': '', 'storage_group': 'minos', 'blocksize': 131072, 'declared': 1248994044.0, 'library': 'CD-LTO4G1', 'si_time': [0.0, 1242939993.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VON262', 'wrapper': 'cpio_odc', 'remaining_bytes': 465168384000L, 'sum_mounts': 156, 'sum_rd_access': 0, 'file_family_width': 1, 'last_access': 1257400773.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0000856', 'capacity_bytes': 858993459200L, 'non_del_files': 858, 'sum_wr_err': 0, 'sum_wr_access': 855, 'file_family': 'minos', 'address': ('131.225.13.59', 7502), 'volume_family': 'minos.minos.cpio_odc', 'modification_time': 1257400773.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1257286976.0}, 'outfilepath': '/pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/711/n13047112_0023_L010185R_D07_r4.cand.dogwood1.0.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 387537400L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7869, 'mc_device': '0,5,1,7', 'compression': 0, 'serial_num': '9310025229', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr209a', 'max_time_in_state': 1200, 'device': 'stkenmvr209a:/dev/rmt/tps0d0n', 'name': 'LTO4_109.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_109.stat', 'mover_address': ('131.225.13.205', 7869), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.205', 42015), 'data_ip': 'stkenmvr209a', 'logname': 'LTO4_109MV', 'hostip': '131.225.13.205', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/711/n13047112_0023_L010185R_D07_r4.cand.dogwood1.0.root', 'uid': 0, 'uname': 'root', 'mtime': 1257400826, 'rmajor': 0, 'machine': ('Linux', 'stkendca21a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16893, 253994256L, 20L, 1, 42411, 5111, 512L, 1257400777, 1257400777, 1257381850), 'mode_octal': '0100036', 'mode': 33188, 'size_bytes': 387537400L, 'gname': 'root', 'gid': 0, 'fullname': '/diskb/write-pool-2/data/000F0000000000000B2471F0', 'inode': 254046704L, 'minor': 20}, 'version': 'v3_8b  CVS $Revision: 1.922 $ <frozen>', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskb/write-pool-2/data/000F0000000000000B2471F0', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/711/.(access)(000F0000000000000B2471F0)', 'fc': {'uid': 0, 'external_label': 'VON262', 'gid': 0, 'location_cookie': '0000_000000000_0000856', 'complete_crc': 3922966897L, 'bfid': 'CDMS125740083300000', 'sanity_cookie': (65536L, 3368846051L), 'size': 387537400L}, 'infilepath': '/diskb/write-pool-2/data/000F0000000000000B2471F0', 'at_the_top': 9, 'local_inode': 1073743790L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.21', 44456), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257400826.097872, 'seek_time': 0.02972102165222168, 'transfer_time': 5.4287691116333008, 't0': 1257400826, 'mount_time': 0.016989946365356445, 'lm_dequeued': 1257400828.1960149, 'job_queued': 1257400826.948622, 'drive_call_time': 0.3128509521484375, 'in_queue': 1.2473349571228027, 'drive_transfer_time': 3.0762121677398682}, 'user_level': 'user2', 'unique_id'(trunc. 4122) Thread tape_thread
00:00:33 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  Done is sent Thread tape_thread
00:00:33 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  maybe_clean Thread tape_thread
00:00:33 stkendca21a.fnal.gov 000378 root I ENCP  Received final dialog for stkendca21a.fnal.gov-1257400826-378-0.  elapsed=7.650sec
00:00:33 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  returned from maybe_clean Thread tape_thread
00:00:33 stkendca21a.fnal.gov 000378 root I ENCP  leaving transfer_file().
00:00:33 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:00:33 stkendca21a.fnal.gov 000378 root I ENCP  pf_status = ('ok', None)  skip_layer_cleanup = False
00:00:33 stkensrv0n.fnal.gov 028796 enstore I FILSRV  Updating  			update file set deleted = 'n', pnfs_path = '/pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/711/n13047112_0023_L010185R_D07_r4.cand.dogwood1.0.root', drive = 'stkenmvr209a:/dev/rmt/tps0d0n:9310025229', pnfs_id = '000F0000000000000B2471F0' where bfid = 'CDMS125740083300000';
00:00:34 stkendca21a.fnal.gov 000378 root I ENCP  MSG_TYPE=ENCP_XFER    write_to_hsm /diskb/write-pool-2/data/000F0000000000000B2471F0 -> /pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/711/n13047112_0023_L010185R_D07_r4.cand.dogwood1.0.root: 387537400 bytes copied to VON262 at 67.9 MB/S (76.4 MB/S network) (120 MB/S drive) (81.7 MB/S disk) mover=LTO4_109.mover drive_id=ULTRIUM-TD4 drive_sn=9310025229 drive_vendor=IBM elapsed=7.9647 {'storage_group': 'minos', 'encp_crc': 3922966897L, 'disk_rate': 81.667389615831269, 'drive_rate': 120.1427095642736, 'driver': 'FTTDriver', 'transfer_rate': 67.867507914886247, 'mover_interface': 'stkenmvr209a', 'overall_rate': 63.430620513381626, 'media_changer': 'SL8500G1.media_changer', 'network_rate': 67.867507914886247, 'encp_ip': '131.225.13.21', 'unique_id': 'stkendca21a.fnal.gov-1257400826-378-0'}
00:00:34 stkendca21a.fnal.gov 000378 root I ENCP  File /pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/711/n13047112_0023_L010185R_D07_r4.cand.dogwood1.0.root copied successfully.
00:00:34 stkendca21a.fnal.gov 000378 root I ENCP  Completed transferring 387537400 bytes in 1 files in 7.96581602097 sec.  Overall rate = 63.4 MB/sec.  Transfer rate = 67.9 MB/sec.  Network rate = 76.4 MB/sec.  Drive rate = 120 MB/sec.  Disk rate = 81.7 MB/sec.  Exit status = 0.
00:00:34 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:00:35 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:00:36 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:00:37 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:00:38 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:00:39 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:00:40 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:00:41 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:00:42 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:00:43 last message repeated 1 times
00:00:43 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:00:44 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:00:44 stkendca19a.fnal.gov 018418 root I ENCP  File /diska/write-pool-1/data/0030000000000000061EB070 transfered for stkendca19a.fnal.gov-1257400742-18418-0 in 18.9425439835 seconds.
00:00:44 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  written bytes 1966080512/1966080512, blocks 15001 header 188 trailer 324 Thread tape_thread
00:00:45 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:00:46 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:00:47 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:00:48 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:00:48 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  filemarks written. Tape VON743 absolute location in blocks 1895478 Thread tape_thread
00:00:48 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  write_tape timing:pull 0.189220666885 write 5.22047615051 crc_check 7.84447598457 freespace 0.206617116928 block_write 13.4607899189 idle 0.327584266663 Thread tape_thread
00:00:48 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  drive stats after write. Tape VON743 position 130 block 0 block_size 0 bloc_loc 1895478 tot_blocks 762939000 BOT 0 read_err 0 write_err 0 bytes 1966080512 block_write_tot 13.4607899189 tape_rate 146059816.983 Thread tape_thread
00:00:48 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  remaining bytes info in DB 545895424000 estimated 543929343488 reported from drive 544050176000 position 131 Thread tape_thread
00:00:48 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  new bitfile request {'uid': 0, 'external_label': 'VON743', 'gid': 0, 'location_cookie': '0000_000000000_0000130', 'complete_crc': 2024348846L, 'sanity_cookie': (65536L, 3826803549L), 'size': 1966080000L} Thread tape_thread
00:00:49 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  set remaining: VON743 544050176000 0000_000000000_0000131 Thread tape_thread
00:00:49 last message repeated 1 times
00:00:49 stkensrv0n.fnal.gov 028762 enstore I VOLSRV  Updating  			update volume set last_access = '2009-11-05 00:00:49', eod_cookie = '0000_000000000_0000131', non_del_files = '132', remaining_bytes = 544050176000 where label = 'VON743';
00:00:49 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  transfer complete volume=VON743 location=131 Thread tape_thread
00:00:49 last message repeated 1 times
00:00:49 stkensrv0n.fnal.gov 028762 enstore I VOLSRV  Updating  			update volume set sum_wr_access = '130', modification_time = '2009-11-05 00:00:49' where label = 'VON743';
00:00:49 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  Sending done to client: {'vc': {'comment': '', 'storage_group': 'exp-db', 'blocksize': 131072, 'declared': 1256853181.0, 'library': 'CD-LTO4G1', 'si_time': [0.0, 1255497368.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VON743', 'wrapper': 'cpio_odc', 'remaining_bytes': 545895424000L, 'sum_mounts': 46, 'sum_rd_access': 0, 'file_family_width': 2, 'last_access': 1257400824.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0000130', 'capacity_bytes': 858993459200L, 'non_del_files': 131, 'sum_wr_err': 0, 'sum_wr_access': 129, 'file_family': 'daily-d0-offline', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.daily-d0-offline.cpio_odc', 'modification_time': 1257400824.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1257340831.0}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702065710_s103260_p1', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 1966080000L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'dismount_delay': 30, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7842, 'mc_device': '0,7,1,12', 'compression': 0, 'serial_num': '1310155269', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr142a', 'max_time_in_state': 1200, 'device': 'stkenmvr142a:/dev/rmt/tps0d0n', 'name': 'LTO4_42.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_42.stat', 'single_filemark': 1, 'mount_delay': 15, 'mover_address': ('131.225.13.170', 7842), 'check_first_written_file': 0, 'callback_addr': ('131.225.13.170', 52222), 'data_ip': 'stkenmvr142a', 'logname': 'LTO4_42MV', 'hostip': '131.225.13.170', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer', 'min_buffer': 67108864L}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702065710_s103260_p1', 'uid': 0, 'uname': 'root', 'mtime': 1257400742, 'rmajor': 0, 'machine': ('Linux', 'stkendca19a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 820618800L, 20L, 1, 1602, 2752, 512L, 1257400711, 1257400711, 1254398418), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 1966080000L, 'gname': 'root', 'gid': 0, 'fullname': '/diska/write-pool-1/data/0030000000000000061EB070', 'inode': 807317616L, 'minor': 20}, 'version': 'v3_8b  CVS $Revision: 1.922 $ <frozen>', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diska/write-pool-1/data/0030000000000000061EB070', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/.(access)(0030000000000000061EB070)', 'fc': {'uid': 0, 'external_label': 'VON743', 'gid': 0, 'location_cookie': '0000_000000000_0000130', 'complete_crc': 2024348846L, 'bfid': 'CDMS125740084800000', 'sanity_cookie': (65536L, 3826803549L), 'size': 1966080000L}, 'infilepath': '/diska/write-pool-1/data/0030000000000000061EB070', 'at_the_top': 379, 'local_inode': 628421102L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.11', 47715), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257400741.3442709, 'seek_time': 0.034271955490112305, 'transfer_time': 23.338196039199829, 't0': 1257400741, 'mount_time': 0.041512966156005859, 'lm_dequeued': 1257400825.655771, 'job_queued': 1257400742.235882, 'drive_call_time': 5.0054464340209961, 'in_queue': 83.419617891311646, 'drive_transfer_time': 18.07538366317749}, 'user_level': 'us(trunc. 4141) Thread tape_thread
00:00:49 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  Done is sent Thread tape_thread
00:00:49 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  maybe_clean Thread tape_thread
00:00:49 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  returned from maybe_clean Thread tape_thread
00:00:49 stkendca19a.fnal.gov 018418 root I ENCP  Received final dialog for stkendca19a.fnal.gov-1257400742-18418-0.  elapsed=107.840sec
00:00:49 stkendca19a.fnal.gov 018418 root I ENCP  leaving transfer_file().
00:00:49 stkendca19a.fnal.gov 018418 root I ENCP  pf_status = ('ok', None)  skip_layer_cleanup = False
00:00:49 stkensrv0n.fnal.gov 028796 enstore I FILSRV  Updating  			update file set deleted = 'n', pnfs_path = '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702065710_s103260_p1', drive = 'stkenmvr142a:/dev/rmt/tps0d0n:1310155269', pnfs_id = '0030000000000000061EB070' where bfid = 'CDMS125740084800000';
00:00:49 stkendca19a.fnal.gov 018418 root I ENCP  MSG_TYPE=ENCP_XFER    write_to_hsm /diska/write-pool-1/data/0030000000000000061EB070 -> /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702065710_s103260_p1: 1966080000 bytes copied to VON743 at 80.3 MB/S (81.4 MB/S network) (104 MB/S drive) (99.1 MB/S disk) mover=LTO4_42.mover drive_id=ULTRIUM-TD4 drive_sn=1310155269 drive_vendor=IBM elapsed=108.11 {'storage_group': 'exp-db', 'encp_crc': 2024348846L, 'disk_rate': 99.145435089918109, 'drive_rate': 103.73223799501868, 'driver': 'FTTDriver', 'transfer_rate': 80.267846907110936, 'mover_interface': 'stkenmvr142a', 'overall_rate': 78.962645752270177, 'media_changer': 'SL8500G1.media_changer', 'network_rate': 80.267846907110936, 'encp_ip': '131.225.13.11', 'unique_id': 'stkendca19a.fnal.gov-1257400742-18418-0'}
00:00:49 stkendca19a.fnal.gov 018418 root I ENCP  File /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702065710_s103260_p1 copied successfully.
00:00:49 stkendca19a.fnal.gov 018418 root I ENCP  Completed transferring 1966080000 bytes in 1 files in 108.111017942 sec.  Overall rate = 79 MB/sec.  Transfer rate = 80.3 MB/sec.  Network rate = 81.4 MB/sec.  Drive rate = 104 MB/sec.  Disk rate = 99.1 MB/sec.  Exit status = 0.
00:00:49 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:00:49 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  write_to_hsm next work on vol=VON743 mover=LTO4_42.mover requester:root
00:00:49 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  HAVE_BOUND:sending write_to_hsm /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702073036_s103270_p20 to mover LTO4_42.mover ('131.225.13.170', 7842) DEL_DISM None
00:00:49 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  MSG_TYPE=ADD_TO_LMQ  request added to work at movers queue for storage group : exp-db
00:00:49 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  HAVE_BOUND:sending {'vc': {'storage_group': 'exp-db', 'external_label': 'VON743', 'library': 'CD-LTO4G1', 'file_family': 'daily-d0-offline', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.daily-d0-offline.cpio_odc', 'file_family_width': 2}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702073036_s103270_p20', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 1701380096L, 'mover': 'LTO4_42.mover', 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702073036_s103270_p20', 'uid': 0, 'uname': 'root', 'type': 'cpio_odc', 'gid': 0, 'mtime': 1257400752, 'rmajor': 0, 'machine': ('Linux', 'stkendca19a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (16877, 820619016L, 20L, 1, 1602, 2752, 512L, 1257400713, 1257400713, 1254398418), 'mode_octal': '0100016', 'mode': 33188, 'gname': 'root', 'size_bytes': 1701380096L, 'fullname': '/diskc/write-pool-3/data/0030000000000000061EB098', 'inode': 807317656L, 'minor': 20}, 'version': 'v3_8b  CVS $Revision: 1.922 $ <frozen>', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskc/write-pool-3/data/0030000000000000061EB098', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/.(access)(0030000000000000061EB098)', 'fc': {'size': 1701380096L, 'external_label': 'VON743', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/diskc/write-pool-3/data/0030000000000000061EB098', 'at_the_top': 361, 'local_inode': 621344939L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.11', 47717), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257400849.6673689, 'encp_start_time': 1257400751.54406, 'job_queued': 1257400752.480011, 't0': 1257400751, 'in_queue': 97.187072038650513}, 'user_level': 'user2', 'unique_id': 'stkendca19a.fnal.gov-1257400752-18852-0'} to mover
00:00:49 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:00:49 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  WRITE_TO_HSM Thread MainThread
00:00:49 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  SENDING {'vc': {'storage_group': 'exp-db', 'external_label': 'VON743', 'library': 'CD-LTO4G1', 'file_family': 'daily-d0-offline', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.daily-d0-offline.cpio_odc', 'file_family_width': 2}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702073036_s103270_p20', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 1701380096L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'dismount_delay': 30, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7842, 'mc_device': '0,7,1,12', 'compression': 0, 'serial_num': '1310155269', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr142a', 'max_time_in_state': 1200, 'device': 'stkenmvr142a:/dev/rmt/tps0d0n', 'name': 'LTO4_42.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_42.stat', 'single_filemark': 1, 'mount_delay': 15, 'mover_address': ('131.225.13.170', 7842), 'check_first_written_file': 0, 'callback_addr': ('131.225.13.170', 52225), 'data_ip': 'stkenmvr142a', 'logname': 'LTO4_42MV', 'hostip': '131.225.13.170', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer', 'min_buffer': 67108864L}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702073036_s103270_p20', 'uid': 0, 'uname': 'root', 'mtime': 1257400752, 'rmajor': 0, 'machine': ('Linux', 'stkendca19a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 820619016L, 20L, 1, 1602, 2752, 512L, 1257400713, 1257400713, 1254398418), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 1701380096L, 'gname': 'root', 'gid': 0, 'fullname': '/diskc/write-pool-3/data/0030000000000000061EB098', 'inode': 807317656L, 'minor': 20}, 'version': 'v3_8b  CVS $Revision: 1.922 $ <frozen>', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskc/write-pool-3/data/0030000000000000061EB098', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/.(access)(0030000000000000061EB098)', 'fc': {'size': 1701380096L, 'external_label': 'VON743', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/diskc/write-pool-3/data/0030000000000000061EB098', 'at_the_top': 361, 'local_inode': 621344939L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.11', 47717), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257400849.6673689, 'encp_start_time': 1257400751.54406, 'job_queued': 1257400752.480011, 't0': 1257400751, 'in_queue': 97.187072038650513}, 'user_level': 'user2', 'unique_id': 'stkendca19a.fnal.gov-1257400752-18852-0'} Thread client_connect_thread
00:00:49 stkendca19a.fnal.gov 018852 root I ENCP  Control socket ('131.225.13.11', 47717) is connected to ('131.225.13.170', 52226) for stkendca19a.fnal.gov-1257400752-18852-0.  elapsed=98.170sec
00:00:49 stkendca19a.fnal.gov 018852 root I ENCP  Received callback ticket from mover LTO4_42.mover for transfer stkendca19a.fnal.gov-1257400752-18852-0.  elapsed=98.170sec
00:00:49 stkendca19a.fnal.gov 018852 root I ENCP  Atempting to connect data socket to mover at ('131.225.13.170', 52225).
00:00:49 stkendca19a.fnal.gov 018852 root I ENCP  Connecting data socket to mover (('131.225.13.170', 52225)) with interface 131.225.13.11.  elapsed=98.170sec
00:00:49 stkendca19a.fnal.gov 018852 root I ENCP  Data socket ('131.225.13.11', 47720) is connected to ('131.225.13.170', 52225).  elapsed=98.170sec
00:00:49 stkendca19a.fnal.gov 018852 root I ENCP  Opening local file.
00:00:49 stkendca19a.fnal.gov 018852 root I ENCP  pf_status = ('ok', None)  skip_layer_cleanup = False
00:00:49 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:00:49 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  tape position: target 131 current 131 0 Thread seek_thread
00:00:49 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  write_tape starting, bytes_to_write=1701380608 Thread tape_thread
00:00:49 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  Write starting. Tape VON743 absolute location in blocks 1895478 Thread tape_thread
00:00:50 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:00:51 last message repeated 1 times
00:00:51 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:00:52 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:00:53 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:00:54 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:00:55 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:00:56 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:00:57 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:00:58 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:00:59 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  skip_fm 185 Thread seek_thread
00:00:59 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  write_tape starting, bytes_to_write=1966080512 Thread tape_thread
00:00:59 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  Write starting. Tape VON266 absolute location in blocks 2604423 Thread tape_thread
00:00:59 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:01:00 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:01:01 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:01:02 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:01:03 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:01:04 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:01:05 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:01:06 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:01:06 stkendca19a.fnal.gov 018852 root I ENCP  File /diskc/write-pool-3/data/0030000000000000061EB098 transfered for stkendca19a.fnal.gov-1257400752-18852-0 in 16.7904980183 seconds.
00:01:06 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  written bytes 1701380608/1701380608, blocks 12981 header 189 trailer 323 Thread tape_thread
00:01:07 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  filemarks written. Tape VON743 absolute location in blocks 1908460 Thread tape_thread
00:01:07 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  write_tape timing:pull 0.18773150444 write 5.38751530647 crc_check 6.87982153893 freespace 0.201356172562 block_write 12.6564245224 idle 0.316411018372 Thread tape_thread
00:01:07 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  drive stats after write. Tape VON743 position 131 block 0 block_size 0 bloc_loc 1908460 tot_blocks 762939000 BOT 0 read_err 0 write_err 0 bytes 1701380608 block_write_tot 12.6564245224 tape_rate 134428219.043 Thread tape_thread
00:01:07 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  remaining bytes info in DB 544050176000 estimated 542348795392 reported from drive 542455808000 position 132 Thread tape_thread
00:01:07 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  new bitfile request {'uid': 0, 'external_label': 'VON743', 'gid': 0, 'location_cookie': '0000_000000000_0000131', 'complete_crc': 351606696L, 'sanity_cookie': (65536L, 2451810617L), 'size': 1701380096L} Thread tape_thread
00:01:07 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  set remaining: VON743 542455808000 0000_000000000_0000132 Thread tape_thread
00:01:07 last message repeated 1 times
00:01:07 stkensrv0n.fnal.gov 028762 enstore I VOLSRV  Updating  			update volume set last_access = '2009-11-05 00:01:07', eod_cookie = '0000_000000000_0000132', non_del_files = '133', remaining_bytes = 542455808000 where label = 'VON743';
00:01:07 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  transfer complete volume=VON743 location=132 Thread tape_thread
00:01:07 last message repeated 1 times
00:01:07 stkensrv0n.fnal.gov 028762 enstore I VOLSRV  Updating  			update volume set sum_wr_access = '131', modification_time = '2009-11-05 00:01:07' where label = 'VON743';
00:01:07 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  Sending done to client: {'vc': {'comment': '', 'storage_group': 'exp-db', 'blocksize': 131072, 'declared': 1256853181.0, 'library': 'CD-LTO4G1', 'si_time': [0.0, 1255497368.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VON743', 'wrapper': 'cpio_odc', 'remaining_bytes': 544050176000L, 'sum_mounts': 46, 'sum_rd_access': 0, 'file_family_width': 2, 'last_access': 1257400849.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0000131', 'capacity_bytes': 858993459200L, 'non_del_files': 132, 'sum_wr_err': 0, 'sum_wr_access': 130, 'file_family': 'daily-d0-offline', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.daily-d0-offline.cpio_odc', 'modification_time': 1257400849.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1257340831.0}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702073036_s103270_p20', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 1701380096L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'dismount_delay': 30, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7842, 'mc_device': '0,7,1,12', 'compression': 0, 'serial_num': '1310155269', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr142a', 'max_time_in_state': 1200, 'device': 'stkenmvr142a:/dev/rmt/tps0d0n', 'name': 'LTO4_42.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_42.stat', 'single_filemark': 1, 'mount_delay': 15, 'mover_address': ('131.225.13.170', 7842), 'check_first_written_file': 0, 'callback_addr': ('131.225.13.170', 52225), 'data_ip': 'stkenmvr142a', 'logname': 'LTO4_42MV', 'hostip': '131.225.13.170', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer', 'min_buffer': 67108864L}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702073036_s103270_p20', 'uid': 0, 'uname': 'root', 'mtime': 1257400752, 'rmajor': 0, 'machine': ('Linux', 'stkendca19a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 820619016L, 20L, 1, 1602, 2752, 512L, 1257400713, 1257400713, 1254398418), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 1701380096L, 'gname': 'root', 'gid': 0, 'fullname': '/diskc/write-pool-3/data/0030000000000000061EB098', 'inode': 807317656L, 'minor': 20}, 'version': 'v3_8b  CVS $Revision: 1.922 $ <frozen>', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskc/write-pool-3/data/0030000000000000061EB098', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/.(access)(0030000000000000061EB098)', 'fc': {'uid': 0, 'external_label': 'VON743', 'gid': 0, 'location_cookie': '0000_000000000_0000131', 'complete_crc': 351606696L, 'bfid': 'CDMS125740086700000', 'sanity_cookie': (65536L, 2451810617L), 'size': 1701380096L}, 'infilepath': '/diskc/write-pool-3/data/0030000000000000061EB098', 'at_the_top': 361, 'local_inode': 621344939L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.11', 47717), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257400751.54406, 'seek_time': 0.034127950668334961, 'transfer_time': 17.787238121032715, 't0': 1257400751, 'mount_time': 0.042963027954101562, 'lm_dequeued': 1257400849.6673689, 'job_queued': 1257400752.480011, 'drive_call_time': 5.1983418464660645, 'in_queue': 97.187072038650513, 'drive_transfer_time': 13.829079866409302}, 'user_level': 'u(trunc. 4142) Thread tape_thread
00:01:07 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  Done is sent Thread tape_thread
00:01:07 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  maybe_clean Thread tape_thread
00:01:07 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  returned from maybe_clean Thread tape_thread
00:01:07 stkendca19a.fnal.gov 018852 root I ENCP  Received final dialog for stkendca19a.fnal.gov-1257400752-18852-0.  elapsed=116.080sec
00:01:07 stkendca19a.fnal.gov 018852 root I ENCP  leaving transfer_file().
00:01:07 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:01:07 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  write_to_hsm next work on vol=VON743 mover=LTO4_42.mover requester:root
00:01:07 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  HAVE_BOUND:sending write_to_hsm /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702070562_s103266_p22 to mover LTO4_42.mover ('131.225.13.170', 7842) DEL_DISM None
00:01:07 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  MSG_TYPE=ADD_TO_LMQ  request added to work at movers queue for storage group : exp-db
00:01:07 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  HAVE_BOUND:sending {'vc': {'storage_group': 'exp-db', 'external_label': 'VON743', 'library': 'CD-LTO4G1', 'file_family': 'daily-d0-offline', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.daily-d0-offline.cpio_odc', 'file_family_width': 2}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702070562_s103266_p22', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 1966080000L, 'mover': 'LTO4_42.mover', 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702070562_s103266_p22', 'uid': 0, 'uname': 'root', 'type': 'cpio_odc', 'gid': 0, 'mtime': 1257400813, 'rmajor': 0, 'machine': ('Linux', 'stkendca17a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (16877, 820619016L, 20L, 1, 1602, 2752, 512L, 1257400777, 1257400777, 1254398418), 'mode_octal': '0100016', 'mode': 33188, 'gname': 'root', 'size_bytes': 1966080000L, 'fullname': '/diska/write-pool-1/data/0030000000000000061EB100', 'inode': 807317760L, 'minor': 20}, 'version': 'v3_8b  CVS $Revision: 1.922 $ <frozen>', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diska/write-pool-1/data/0030000000000000061EB100', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/.(access)(0030000000000000061EB100)', 'fc': {'size': 1966080000L, 'external_label': 'VON743', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/diska/write-pool-1/data/0030000000000000061EB100', 'at_the_top': 281, 'local_inode': 268435937L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.5', 44347), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257400867.6867249, 'encp_start_time': 1257400812.029377, 'job_queued': 1257400814.1018741, 't0': 1257400812, 'in_queue': 53.5845627784729}, 'user_level': 'user2', 'unique_id': 'stkendca17a.fnal.gov-1257400813-30753-0'} to mover
00:01:07 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:01:07 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  WRITE_TO_HSM Thread MainThread
00:01:07 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  SENDING {'vc': {'storage_group': 'exp-db', 'external_label': 'VON743', 'library': 'CD-LTO4G1', 'file_family': 'daily-d0-offline', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.daily-d0-offline.cpio_odc', 'file_family_width': 2}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702070562_s103266_p22', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 1966080000L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'dismount_delay': 30, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7842, 'mc_device': '0,7,1,12', 'compression': 0, 'serial_num': '1310155269', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr142a', 'max_time_in_state': 1200, 'device': 'stkenmvr142a:/dev/rmt/tps0d0n', 'name': 'LTO4_42.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_42.stat', 'single_filemark': 1, 'mount_delay': 15, 'mover_address': ('131.225.13.170', 7842), 'check_first_written_file': 0, 'callback_addr': ('131.225.13.170', 52227), 'data_ip': 'stkenmvr142a', 'logname': 'LTO4_42MV', 'hostip': '131.225.13.170', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer', 'min_buffer': 67108864L}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702070562_s103266_p22', 'uid': 0, 'uname': 'root', 'mtime': 1257400813, 'rmajor': 0, 'machine': ('Linux', 'stkendca17a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 820619016L, 20L, 1, 1602, 2752, 512L, 1257400777, 1257400777, 1254398418), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 1966080000L, 'gname': 'root', 'gid': 0, 'fullname': '/diska/write-pool-1/data/0030000000000000061EB100', 'inode': 807317760L, 'minor': 20}, 'version': 'v3_8b  CVS $Revision: 1.922 $ <frozen>', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diska/write-pool-1/data/0030000000000000061EB100', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/.(access)(0030000000000000061EB100)', 'fc': {'size': 1966080000L, 'external_label': 'VON743', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/diska/write-pool-1/data/0030000000000000061EB100', 'at_the_top': 281, 'local_inode': 268435937L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.5', 44347), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257400867.6867249, 'encp_start_time': 1257400812.029377, 'job_queued': 1257400814.1018741, 't0': 1257400812, 'in_queue': 53.5845627784729}, 'user_level': 'user2', 'unique_id': 'stkendca17a.fnal.gov-1257400813-30753-0'} Thread client_connect_thread
00:01:07 stkendca19a.fnal.gov 018852 root I ENCP  pf_status = ('ok', None)  skip_layer_cleanup = False
00:01:07 stkendca17a.fnal.gov 030753 root I ENCP  Control socket ('131.225.13.5', 44347) is connected to ('131.225.13.170', 52228) for stkendca17a.fnal.gov-1257400813-30753-0.  elapsed=55.710sec
00:01:07 stkendca17a.fnal.gov 030753 root I ENCP  Received callback ticket from mover LTO4_42.mover for transfer stkendca17a.fnal.gov-1257400813-30753-0.  elapsed=55.720sec
00:01:07 stkendca17a.fnal.gov 030753 root I ENCP  Atempting to connect data socket to mover at ('131.225.13.170', 52227).
00:01:07 stkendca17a.fnal.gov 030753 root I ENCP  Connecting data socket to mover (('131.225.13.170', 52227)) with interface 131.225.13.5.  elapsed=55.720sec
00:01:07 stkendca17a.fnal.gov 030753 root I ENCP  Data socket ('131.225.13.5', 44351) is connected to ('131.225.13.170', 52227).  elapsed=55.720sec
00:01:07 stkendca17a.fnal.gov 030753 root I ENCP  Opening local file.
00:01:07 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:01:07 stkendca17a.fnal.gov 030753 root I ENCP  pf_status = ('ok', None)  skip_layer_cleanup = False
00:01:07 stkensrv0n.fnal.gov 028796 enstore I FILSRV  Updating  			update file set deleted = 'n', pnfs_path = '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702073036_s103270_p20', drive = 'stkenmvr142a:/dev/rmt/tps0d0n:1310155269', pnfs_id = '0030000000000000061EB098' where bfid = 'CDMS125740086700000';
00:01:07 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  tape position: target 132 current 132 0 Thread seek_thread
00:01:07 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  write_tape starting, bytes_to_write=1966080512 Thread tape_thread
00:01:07 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  Write starting. Tape VON743 absolute location in blocks 1908460 Thread tape_thread
00:01:07 stkendca19a.fnal.gov 018852 root I ENCP  MSG_TYPE=ENCP_XFER    write_to_hsm /diskc/write-pool-3/data/0030000000000000061EB098 -> /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702073036_s103270_p20: 1701380096 bytes copied to VON743 at 91.1 MB/S (78.8 MB/S network) (117 MB/S drive) (96.7 MB/S disk) mover=LTO4_42.mover drive_id=ULTRIUM-TD4 drive_sn=1310155269 drive_vendor=IBM elapsed=116.35 {'storage_group': 'exp-db', 'encp_crc': 351606696L, 'disk_rate': 96.693686023382142, 'drive_rate': 117.32975119633146, 'driver': 'FTTDriver', 'transfer_rate': 91.101252604670776, 'mover_interface': 'stkenmvr142a', 'overall_rate': 89.278223037178464, 'media_changer': 'SL8500G1.media_changer', 'network_rate': 91.101252604670776, 'encp_ip': '131.225.13.11', 'unique_id': 'stkendca19a.fnal.gov-1257400752-18852-0'}
00:01:07 stkendca19a.fnal.gov 018852 root I ENCP  File /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702073036_s103270_p20 copied successfully.
00:01:07 stkendca19a.fnal.gov 018852 root I ENCP  Completed transferring 1701380096 bytes in 1 files in 116.350986004 sec.  Overall rate = 89.3 MB/sec.  Transfer rate = 91.1 MB/sec.  Network rate = 78.8 MB/sec.  Drive rate = 117 MB/sec.  Disk rate = 96.7 MB/sec.  Exit status = 0.
00:01:08 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:01:09 last message repeated 1 times
00:01:09 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:01:10 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:01:11 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:01:12 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:01:13 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:01:13 stkensrv4n.fnal.gov 005357 enstore E LTO4G1LM  MOVER ERROR RQ {'status': ('ok', None), 'returned_work': None, 'library': None, 'mover_type': '', 'current_location': '0000_000000000_0000000', 'ip_map': '', 'operation': None, 'read_only': 0, 'current_time': 1257400873.858429, 'error_source': None, 'external_label': None, 'state': 'OFFLINE', 'r_a': (('131.225.13.194', 54880), 16279L, '131.225.13.194-54880-1256586929.509013-4866'), 'library_list': [('CD-LTO4G1.library_manager', ('131.225.13.4', 8541))], 'current_priority': (None, None), 'transfer_deficiency': 1, 'time_in_state': 813943.49837088585, 'address': ('131.225.13.194', 7860), 'volume_family': None, 'work': 'mover_error', 'mover': 'LTO4_99.mover', 'volume_status': (['none', 'none'], ['none', 'none']), 'unique_id': None}
00:01:14 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:01:15 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:01:15 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:01:15 stkendca18a.fnal.gov 021252 root I ENCP  Version: v3_8b  CVS $Revision: 1.922 $ <frozen>  OS: Linux 2.6.9-89.0.9.ELsmp i686 Release:  Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0)  Group: root(0)  Euser: root(0)  Egroup: root(0)  Library: CD-LTO4G1  Storage Group: exp-db  File Family: daily-d0-offline  FF Wrapper: cpio_odc  FF Width: 2  Current working directory: stkendca18a.fnal.gov:/home/enstore/dcache-code/dcache-fermi-config  Command line: /fnal/ups/prd/encp/v3_8b/Linux-2-6-dcache/encp --verbose 4 --threaded --ecrc --bypass-filesystem-max-filesize-check --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702070562_s103266_p25 --put-cache 0030000000000000061EB170 /diska/write-pool-1/data/0030000000000000061EB170
00:01:15 stkendca18a.fnal.gov 021257 root I ENCP  Version: v3_8b  CVS $Revision: 1.922 $ <frozen>  OS: Linux 2.6.9-89.0.9.ELsmp i686 Release:  Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0)  Group: root(0)  Euser: root(0)  Egroup: root(0)  Library: CD-LTO4G1  Storage Group: exp-db  File Family: daily-d0-offline  FF Wrapper: cpio_odc  FF Width: 2  Current working directory: stkendca18a.fnal.gov:/home/enstore/dcache-code/dcache-fermi-config  Command line: /fnal/ups/prd/encp/v3_8b/Linux-2-6-dcache/encp --verbose 4 --threaded --ecrc --bypass-filesystem-max-filesize-check --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702075233_s103280_p2 --put-cache 0030000000000000061EB148 /diska/write-pool-1/data/0030000000000000061EB148
00:01:16 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:01:16 stkendca18a.fnal.gov 021252 root I ENCP  Creating zero length output files.
00:01:16 stkendca18a.fnal.gov 021252 root I ENCP  Sending /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702070562_s103266_p25 write request to LM: unique_id: stkendca18a.fnal.gov-1257400876-21252-0 inputfile: /diska/write-pool-1/data/0030000000000000061EB170 outputfile: /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/.(access)(0030000000000000061EB170)
00:01:16 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  MSG_TYPE=ADD_TO_LMQ  request added to pending queue for storage group : exp-db
00:01:16 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  write Q'd /diska/write-pool-1/data/0030000000000000061EB170 (stkendca18a.fnal.gov-1257400876-21252-0) -> /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702070562_s103266_p25 : library=CD-LTO4G1 family=daily-d0-offline requester:root volume_family:exp-db.daily-d0-offline.cpio_odc
00:01:16 stkendca18a.fnal.gov 021252 root I ENCP  Listening for control socket at: ('131.225.13.6', 54459)
00:01:16 stkendca18a.fnal.gov 021252 root I ENCP  Waiting for mover to connect control socket at address: ('131.225.13.6', 54459)  elapsed=1.800sec
00:01:16 stkendca18a.fnal.gov 021257 root I ENCP  Creating zero length output files.
00:01:16 stkendca18a.fnal.gov 021257 root I ENCP  Sending /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702075233_s103280_p2 write request to LM: unique_id: stkendca18a.fnal.gov-1257400876-21257-0 inputfile: /diska/write-pool-1/data/0030000000000000061EB148 outputfile: /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/.(access)(0030000000000000061EB148)
00:01:16 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  MSG_TYPE=ADD_TO_LMQ  request added to pending queue for storage group : exp-db
00:01:16 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  write Q'd /diska/write-pool-1/data/0030000000000000061EB148 (stkendca18a.fnal.gov-1257400876-21257-0) -> /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702075233_s103280_p2 : library=CD-LTO4G1 family=daily-d0-offline requester:root volume_family:exp-db.daily-d0-offline.cpio_odc
00:01:16 stkendca18a.fnal.gov 021257 root I ENCP  Listening for control socket at: ('131.225.13.6', 54460)
00:01:16 stkendca18a.fnal.gov 021257 root I ENCP  Waiting for mover to connect control socket at address: ('131.225.13.6', 54460)  elapsed=1.930sec
00:01:17 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:01:18 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:01:19 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:01:20 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:01:21 stkendca18a.fnal.gov 019668 root I ENCP  File /diskb/write-pool-2/data/0030000000000000061EAEC8 transfered for stkendca18a.fnal.gov-1257400567-19668-0 in 21.4798440933 seconds.
00:01:21 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:01:21 stkendca22a.fnal.gov 016543 root I ENCP  Version: v3_8b  CVS $Revision: 1.922 $ <frozen>  OS: Linux 2.6.9-89.0.9.ELsmp i686 Release:  Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0)  Group: root(0)  Euser: root(0)  Egroup: root(0)  Library: garbage  Storage Group: test  File Family: dcache  FF Wrapper: cpio_odc  FF Width: 1  Current working directory: stkendca22a.fnal.gov:/home/enstore/dcache-log  Command line: /fnal/ups/prd/encp/v3_8b/Linux-2-6-dcache/encp --verbose 4 --threaded --ecrc --bypass-filesystem-max-filesize-check --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0003 --put-cache 0005000000000000005FA5F8 /diskb/write-pool-2/data/0005000000000000005FA5F8
00:01:21 stkendca22a.fnal.gov 016637 root I ENCP  Version: v3_8b  CVS $Revision: 1.922 $ <frozen>  OS: Linux 2.6.9-89.0.9.ELsmp i686 Release:  Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0)  Group: root(0)  Euser: root(0)  Egroup: root(0)  Library: garbage  Storage Group: test  File Family: dcache  FF Wrapper: cpio_odc  FF Width: 1  Current working directory: stkendca22a.fnal.gov:/home/enstore/dcache-log  Command line: /fnal/ups/prd/encp/v3_8b/Linux-2-6-dcache/encp --verbose 4 --threaded --ecrc --bypass-filesystem-max-filesize-check --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0001 --put-cache 0005000000000000005FA590 /diskb/write-pool-2/data/0005000000000000005FA590
00:01:21 stkendca22a.fnal.gov 016653 root I ENCP  Version: v3_8b  CVS $Revision: 1.922 $ <frozen>  OS: Linux 2.6.9-89.0.9.ELsmp i686 Release:  Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0)  Group: root(0)  Euser: root(0)  Egroup: root(0)  Library: garbage  Storage Group: test  File Family: dcache  FF Wrapper: cpio_odc  FF Width: 1  Current working directory: stkendca22a.fnal.gov:/home/enstore/dcache-log  Command line: /fnal/ups/prd/encp/v3_8b/Linux-2-6-dcache/encp --verbose 4 --threaded --ecrc --bypass-filesystem-max-filesize-check --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0002 --put-cache 0005000000000000005FA5C8 /diskb/write-pool-2/data/0005000000000000005FA5C8
00:01:22 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:01:22 stkendca22a.fnal.gov 016543 root I ENCP  Configuration Server: no such name: 'garbage.library_manager'
00:01:22 stkendca22a.fnal.gov 016543 root E ENCP  INFILE=/diskb/write-pool-2/data/0005000000000000005FA5F8 OUTFILE=/pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0003 FILESIZE=801512 LABEL= LOCATION= DRIVE= DRIVE_SN= TRANSFER_TIME=0.00 SEEK_TIME=0.00 MOUNT_TIME=0.00 QWAIT_TIME=0.00 TIME2NOW=0.90 CRC= STATUS=KEYERROR Configuration Server: no such name: 'garbage.library_manager'
00:01:22 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:01:22 stkendca22a.fnal.gov 016637 root I ENCP  Configuration Server: no such name: 'garbage.library_manager'
00:01:22 stkendca22a.fnal.gov 016637 root E ENCP  INFILE=/diskb/write-pool-2/data/0005000000000000005FA590 OUTFILE=/pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0001 FILESIZE=801512 LABEL= LOCATION= DRIVE= DRIVE_SN= TRANSFER_TIME=0.00 SEEK_TIME=0.00 MOUNT_TIME=0.00 QWAIT_TIME=0.00 TIME2NOW=0.90 CRC= STATUS=KEYERROR Configuration Server: no such name: 'garbage.library_manager'
00:01:22 stkendca22a.fnal.gov 016653 root I ENCP  Configuration Server: no such name: 'garbage.library_manager'
00:01:22 stkendca22a.fnal.gov 016653 root E ENCP  INFILE=/diskb/write-pool-2/data/0005000000000000005FA5C8 OUTFILE=/pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0002 FILESIZE=801512 LABEL= LOCATION= DRIVE= DRIVE_SN= TRANSFER_TIME=0.00 SEEK_TIME=0.00 MOUNT_TIME=0.00 QWAIT_TIME=0.00 TIME2NOW=0.88 CRC= STATUS=KEYERROR Configuration Server: no such name: 'garbage.library_manager'
00:01:23 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:01:24 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:01:25 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  written bytes 1966080512/1966080512, blocks 15001 header 188 trailer 324 Thread tape_thread
00:01:25 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:01:26 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:01:26 stkenmvr10a.fnal.gov 005910 root I DBT10MV  read bytes 1891545456/1891545456, blocks 14432 header 0 Thread tape_thread
00:01:26 stkenmvr10a.fnal.gov 005910 root I DBT10MV  write_client: wrote 1891545456/1891545456 bytes Thread net_thread
00:01:26 stkenmvr10a.fnal.gov 005910 root I DBT10MV  write_client waits for tape thread to exit Thread net_thread
00:01:26 stkenmvr10a.fnal.gov 005910 root I DBT10MV  drive stats after read. Tape VO6332 position 11 block 14432 block_size 0 bloc_loc 164157 tot_blocks 0 BOT 0 read_err 0 write_err 0 bytes 1891545456 block_read_tot 59.0644016266 tape_rate 32025135.3422 Thread tape_thread
00:01:26 stkenmvr10a.fnal.gov 005910 root I DBT10MV  read_tape exiting, read 1891545456/1891545456 bytes Thread tape_thread
00:01:26 stkenmvr10a.fnal.gov 005910 root I DBT10MV  read_tape timing:get_space 0.312160253525 read 52.0308687687 crc_check 6.46894884109 push 0.252423763275 block_read 59.0644016266 idle 0.553084373474 Thread tape_thread
00:01:26 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  filemarks written. Tape VON266 absolute location in blocks 2619425 Thread tape_thread
00:01:26 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  write_tape timing:pull 0.168275594711 write 8.47290587425 crc_check 9.07478642464 freespace 0.191688537598 block_write 17.9076564312 idle 0.300625562668 Thread tape_thread
00:01:26 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  drive stats after write. Tape VON266 position 185 block 0 block_size 0 bloc_loc 2619425 tot_blocks 762939000 BOT 0 read_err 0 write_err 0 bytes 1966080512 block_write_tot 17.9076564312 tape_rate 109789939.267 Thread tape_thread
00:01:26 stkendca21a.fnal.gov 001465 root I ENCP  Version: v3_8b  CVS $Revision: 1.922 $ <frozen>  OS: Linux 2.6.9-89.0.9.ELsmp i686 Release:  Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0)  Group: root(0)  Euser: root(0)  Egroup: root(0)  Library: CD-LTO4G1  Storage Group: minos  File Family: minos  FF Wrapper: cpio_odc  FF Width: 1  Current working directory: stkendca21a.fnal.gov:/  Command line: /fnal/ups/prd/encp/v3_8b/Linux-2-6-dcache/encp --verbose 4 --threaded --ecrc --bypass-filesystem-max-filesize-check --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/713/n13047134_0023_L010185R_D07_r4.cand.dogwood1.0.root --put-cache 000F0000000000000B247230 /diskb/write-pool-2/data/000F0000000000000B247230
00:01:26 stkendca21a.fnal.gov 001456 root I ENCP  Version: v3_8b  CVS $Revision: 1.922 $ <frozen>  OS: Linux 2.6.9-89.0.9.ELsmp i686 Release:  Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0)  Group: root(0)  Euser: root(0)  Egroup: root(0)  Library: CD-LTO4G1  Storage Group: minos  File Family: minos  FF Wrapper: cpio_odc  FF Width: 1  Current working directory: stkendca21a.fnal.gov:/  Command line: /fnal/ups/prd/encp/v3_8b/Linux-2-6-dcache/encp --verbose 4 --threaded --ecrc --bypass-filesystem-max-filesize-check --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/712/n13047129_0017_L010185R_D07_r4.cand.dogwood1.0.root --put-cache 000F0000000000000B247260 /diskb/write-pool-2/data/000F0000000000000B247260
00:01:26 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  remaining bytes info in DB 457628672000 estimated 455662591488 reported from drive 455795712000 position 186 Thread tape_thread
00:01:26 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  new bitfile request {'uid': 0, 'external_label': 'VON266', 'gid': 0, 'location_cookie': '0000_000000000_0000185', 'complete_crc': 4067638873L, 'sanity_cookie': (65536L, 1970242448L), 'size': 1966080000L} Thread tape_thread
00:01:26 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  set remaining: VON266 455795712000 0000_000000000_0000186 Thread tape_thread
00:01:26 last message repeated 1 times
00:01:26 stkensrv0n.fnal.gov 028762 enstore I VOLSRV  Updating  			update volume set last_access = '2009-11-05 00:01:26', eod_cookie = '0000_000000000_0000186', non_del_files = '186', remaining_bytes = 455795712000 where label = 'VON266';
00:01:26 stkendca21a.fnal.gov 001465 root I ENCP  Creating zero length output files.
00:01:27 stkendm7a.fnal.gov 031675 root I ENCP  File /pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKE0/QKE090/f0 transfered for stkendm7a.fnal.gov-1257400491-31675-3 in 61.5279741287 seconds. Thread READ
00:01:27 stkendca21a.fnal.gov 001465 root I ENCP  Sending /pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/713/n13047134_0023_L010185R_D07_r4.cand.dogwood1.0.root write request to LM: unique_id: stkendca21a.fnal.gov-1257400886-1465-0 inputfile: /diskb/write-pool-2/data/000F0000000000000B247230 outputfile: /pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/713/.(access)(000F0000000000000B247230)
00:01:27 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  transfer complete volume=VON266 location=186 Thread tape_thread
00:01:27 last message repeated 1 times
00:01:27 stkensrv0n.fnal.gov 028762 enstore I VOLSRV  Updating  			update volume set sum_wr_access = '185', last_access = '2009-11-05 00:01:27', modification_time = '2009-11-05 00:01:27' where label = 'VON266';
00:01:27 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  MSG_TYPE=ADD_TO_LMQ  request added to pending queue for storage group : minos
00:01:27 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  write Q'd /diskb/write-pool-2/data/000F0000000000000B247230 (stkendca21a.fnal.gov-1257400886-1465-0) -> /pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/713/n13047134_0023_L010185R_D07_r4.cand.dogwood1.0.root : library=CD-LTO4G1 family=minos requester:root volume_family:minos.minos.cpio_odc
00:01:27 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  Sending done to client: {'vc': {'comment': '', 'storage_group': 'exp-db', 'blocksize': 131072, 'declared': 1248994048.0, 'library': 'CD-LTO4G1', 'si_time': [0.0, 1242744483.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VON266', 'wrapper': 'cpio_odc', 'remaining_bytes': 457628672000L, 'sum_mounts': 72, 'sum_rd_access': 0, 'file_family_width': 2, 'last_access': 1257400505.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0000185', 'capacity_bytes': 858993459200L, 'non_del_files': 185, 'sum_wr_err': 0, 'sum_wr_access': 184, 'file_family': 'daily-d0-offline', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.daily-d0-offline.cpio_odc', 'modification_time': 1257400505.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1257365852.0}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702074763_s103275_p3', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 1966080000L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7865, 'mc_device': '0,6,1,1', 'compression': 0, 'serial_num': '9310801322', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr205a', 'max_time_in_state': 1200, 'device': 'stkenmvr205a:/dev/rmt/tps0d0n', 'name': 'LTO4_105.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_105.stat', 'mover_address': ('131.225.13.201', 7865), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.201', 42191), 'data_ip': 'stkenmvr205a', 'logname': 'LTO4_105MV', 'hostip': '131.225.13.201', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702074763_s103275_p3', 'uid': 0, 'uname': 'root', 'mtime': 1257400567, 'rmajor': 0, 'machine': ('Linux', 'stkendca18a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 820619016L, 20L, 1, 1602, 2752, 512L, 1257400558, 1257400558, 1254398418), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 1966080000L, 'gname': 'root', 'gid': 0, 'fullname': '/diskb/write-pool-2/data/0030000000000000061EAEC8', 'inode': 807317192L, 'minor': 20}, 'version': 'v3_8b  CVS $Revision: 1.922 $ <frozen>', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskb/write-pool-2/data/0030000000000000061EAEC8', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/.(access)(0030000000000000061EAEC8)', 'fc': {'uid': 0, 'external_label': 'VON266', 'gid': 0, 'location_cookie': '0000_000000000_0000185', 'complete_crc': 4067638873L, 'bfid': 'CDMS125740088600000', 'sanity_cookie': (65536L, 1970242448L), 'size': 1966080000L}, 'infilepath': '/diskb/write-pool-2/data/0030000000000000061EAEC8', 'at_the_top': 944, 'local_inode': 536875141L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.6', 54441), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257400566.393677, 'seek_time': 153.49767017364502, 'transfer_time': 27.380344867706299, 't0': 1257400566, 'lm_dequeued': 1257400706.0882299, 'job_queued': 1257400567.9400539, 'drive_call_time': 8.268531322479248, 'in_queue': 138.14785313606262, 'drive_transfer_time': 19.923529148101807}, 'user_level': 'user2', 'unique_id': 'stkendca18a.fnal.gov-1257400567-19668-0'} Thread tape_thread
00:01:27 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  Done is sent Thread tape_thread
00:01:27 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  maybe_clean Thread tape_thread
00:01:27 stkendca21a.fnal.gov 001456 root I ENCP  Creating zero length output files.
00:01:27 stkendca21a.fnal.gov 001465 root I ENCP  Listening for control socket at: ('131.225.13.21', 44463)
00:01:27 stkendca21a.fnal.gov 001465 root I ENCP  Waiting for mover to connect control socket at address: ('131.225.13.21', 44463)  elapsed=0.880sec
00:01:27 stkendca21a.fnal.gov 001456 root I ENCP  Sending /pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/712/n13047129_0017_L010185R_D07_r4.cand.dogwood1.0.root write request to LM: unique_id: stkendca21a.fnal.gov-1257400886-1456-0 inputfile: /diskb/write-pool-2/data/000F0000000000000B247260 outputfile: /pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/712/.(access)(000F0000000000000B247260)
00:01:27 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  returned from maybe_clean Thread tape_thread
00:01:27 stkendca18a.fnal.gov 019668 root I ENCP  Received final dialog for stkendca18a.fnal.gov-1257400567-19668-0.  elapsed=320.670sec
00:01:27 stkendca18a.fnal.gov 019668 root I ENCP  leaving transfer_file().
00:01:27 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  MSG_TYPE=ADD_TO_LMQ  request added to pending queue for storage group : minos
00:01:27 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  write Q'd /diskb/write-pool-2/data/000F0000000000000B247260 (stkendca21a.fnal.gov-1257400886-1456-0) -> /pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/712/n13047129_0017_L010185R_D07_r4.cand.dogwood1.0.root : library=CD-LTO4G1 family=minos requester:root volume_family:minos.minos.cpio_odc
00:01:27 stkendca21a.fnal.gov 001456 root I ENCP  Listening for control socket at: ('131.225.13.21', 44464)
00:01:27 stkendca21a.fnal.gov 001456 root I ENCP  Waiting for mover to connect control socket at address: ('131.225.13.21', 44464)  elapsed=0.920sec
00:01:27 stkendca18a.fnal.gov 019668 root I ENCP  pf_status = ('ok', None)  skip_layer_cleanup = False
00:01:27 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:01:27 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  write_to_hsm next work on vol=VON266 mover=LTO4_105.mover requester:root
00:01:27 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  HAVE_BOUND:sending write_to_hsm /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702068213_s103263_p28 to mover LTO4_105.mover ('131.225.13.201', 7865) DEL_DISM None
00:01:27 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  MSG_TYPE=ADD_TO_LMQ  request added to work at movers queue for storage group : exp-db
00:01:27 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  HAVE_BOUND:sending {'vc': {'storage_group': 'exp-db', 'external_label': 'VON266', 'library': 'CD-LTO4G1', 'file_family': 'daily-d0-offline', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.daily-d0-offline.cpio_odc', 'file_family_width': 2}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702068213_s103263_p28', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 1966080000L, 'mover': 'LTO4_105.mover', 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702068213_s103263_p28', 'uid': 0, 'uname': 'root', 'type': 'cpio_odc', 'gid': 0, 'mtime': 1257400814, 'rmajor': 0, 'machine': ('Linux', 'stkendca18a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (16877, 820618800L, 20L, 1, 1602, 2752, 512L, 1257400772, 1257400772, 1254398418), 'mode_octal': '0100016', 'mode': 33188, 'gname': 'root', 'size_bytes': 1966080000L, 'fullname': '/diskc/write-pool-3/data/0030000000000000061EB0D8', 'inode': 807317720L, 'minor': 20}, 'version': 'v3_8b  CVS $Revision: 1.922 $ <frozen>', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskc/write-pool-3/data/0030000000000000061EB0D8', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/.(access)(0030000000000000061EB0D8)', 'fc': {'size': 1966080000L, 'external_label': 'VON266', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/diskc/write-pool-3/data/0030000000000000061EB0D8', 'at_the_top': 302, 'local_inode': 536876407L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.6', 54453), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257400887.20081, 'encp_start_time': 1257400813.491024, 'job_queued': 1257400814.430378, 't0': 1257400813, 'in_queue': 72.770142078399658}, 'user_level': 'user2', 'unique_id': 'stkendca18a.fnal.gov-1257400814-20159-0'} to mover
00:01:27 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:01:27 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  WRITE_TO_HSM Thread MainThread
00:01:27 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  SENDING {'vc': {'storage_group': 'exp-db', 'external_label': 'VON266', 'library': 'CD-LTO4G1', 'file_family': 'daily-d0-offline', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.daily-d0-offline.cpio_odc', 'file_family_width': 2}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702068213_s103263_p28', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 1966080000L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7865, 'mc_device': '0,6,1,1', 'compression': 0, 'serial_num': '9310801322', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr205a', 'max_time_in_state': 1200, 'device': 'stkenmvr205a:/dev/rmt/tps0d0n', 'name': 'LTO4_105.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_105.stat', 'mover_address': ('131.225.13.201', 7865), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.201', 42200), 'data_ip': 'stkenmvr205a', 'logname': 'LTO4_105MV', 'hostip': '131.225.13.201', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702068213_s103263_p28', 'uid': 0, 'uname': 'root', 'mtime': 1257400814, 'rmajor': 0, 'machine': ('Linux', 'stkendca18a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 820618800L, 20L, 1, 1602, 2752, 512L, 1257400772, 1257400772, 1254398418), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 1966080000L, 'gname': 'root', 'gid': 0, 'fullname': '/diskc/write-pool-3/data/0030000000000000061EB0D8', 'inode': 807317720L, 'minor': 20}, 'version': 'v3_8b  CVS $Revision: 1.922 $ <frozen>', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskc/write-pool-3/data/0030000000000000061EB0D8', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/.(access)(0030000000000000061EB0D8)', 'fc': {'size': 1966080000L, 'external_label': 'VON266', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/diskc/write-pool-3/data/0030000000000000061EB0D8', 'at_the_top': 302, 'local_inode': 536876407L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.6', 54453), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257400887.20081, 'encp_start_time': 1257400813.491024, 'job_queued': 1257400814.430378, 't0': 1257400813, 'in_queue': 72.770142078399658}, 'user_level': 'user2', 'unique_id': 'stkendca18a.fnal.gov-1257400814-20159-0'} Thread client_connect_thread
00:01:27 stkendca18a.fnal.gov 020159 root I ENCP  Control socket ('131.225.13.6', 54453) is connected to ('131.225.13.201', 42201) for stkendca18a.fnal.gov-1257400814-20159-0.  elapsed=73.760sec
00:01:27 stkendca18a.fnal.gov 020159 root I ENCP  Received callback ticket from mover LTO4_105.mover for transfer stkendca18a.fnal.gov-1257400814-20159-0.  elapsed=73.770sec
00:01:27 stkendca18a.fnal.gov 020159 root I ENCP  Atempting to connect data socket to mover at ('131.225.13.201', 42200).
00:01:27 stkendca18a.fnal.gov 020159 root I ENCP  Connecting data socket to mover (('131.225.13.201', 42200)) with interface 131.225.13.6.  elapsed=73.770sec
00:01:27 stkendca18a.fnal.gov 020159 root I ENCP  Data socket ('131.225.13.6', 54461) is connected to ('131.225.13.201', 42200).  elapsed=73.770sec
00:01:27 stkendca18a.fnal.gov 020159 root I ENCP  Opening local file.
00:01:27 stkensrv0n.fnal.gov 028796 enstore I FILSRV  Updating  			update file set deleted = 'n', pnfs_path = '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702074763_s103275_p3', drive = 'stkenmvr205a:/dev/rmt/tps0d0n:9310801322', pnfs_id = '0030000000000000061EAEC8' where bfid = 'CDMS125740088600000';
00:01:27 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  tape position: target 186 current 186 0 Thread seek_thread
00:01:27 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  write_tape starting, bytes_to_write=1966080512 Thread tape_thread
00:01:27 stkendca18a.fnal.gov 020159 root I ENCP  pf_status = ('ok', None)  skip_layer_cleanup = False
00:01:27 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  Write starting. Tape VON266 absolute location in blocks 2619425 Thread tape_thread
00:01:27 stkendca18a.fnal.gov 019668 root I ENCP  MSG_TYPE=ENCP_XFER    write_to_hsm /diskb/write-pool-2/data/0030000000000000061EAEC8 -> /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702074763_s103275_p3: 1966080000 bytes copied to VON266 at 68.4 MB/S (81 MB/S network) (94.1 MB/S drive) (87.5 MB/S disk) mover=LTO4_105.mover drive_id=ULTRIUM-TD4 drive_sn=9310801322 drive_vendor=IBM elapsed=320.97 {'storage_group': 'exp-db', 'encp_crc': 4067638873L, 'disk_rate': 87.499760834067374, 'drive_rate': 94.109832954902899, 'driver': 'FTTDriver', 'transfer_rate': 68.419497531439021, 'mover_interface': 'stkenmvr205a', 'overall_rate': 10.346281421485861, 'media_changer': 'SL8500G1.media_changer', 'network_rate': 68.419497531439021, 'encp_ip': '131.225.13.6', 'unique_id': 'stkendca18a.fnal.gov-1257400567-19668-0'}
00:01:27 stkendca18a.fnal.gov 019668 root I ENCP  File /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702074763_s103275_p3 copied successfully.
00:01:27 stkendca18a.fnal.gov 019668 root I ENCP  Completed transferring 1966080000 bytes in 1 files in 320.976701021 sec.  Overall rate = 10.3 MB/sec.  Transfer rate = 68.4 MB/sec.  Network rate = 81 MB/sec.  Drive rate = 94.1 MB/sec.  Disk rate = 87.5 MB/sec.  Exit status = 0.
00:01:27 stkenmvr10a.fnal.gov 005910 root I DBT10MV  write_client exits Thread net_thread
00:01:27 stkenmvr10a.fnal.gov 005910 root I DBT10MV  transfer complete volume=VO6332 location=11 Thread net_thread
00:01:27 last message repeated 1 times
00:01:27 stkensrv0n.fnal.gov 028762 enstore I VOLSRV  Updating  			update volume set sum_rd_access = '80', last_access = '2009-11-05 00:01:27' where label = 'VO6332';
00:01:27 stkenmvr10a.fnal.gov 005910 root I DBT10MV  Sending done to client: {'vc': {'comment': '', 'declared': 1083263100.0, 'blocksize': 131072, 'storage_group': 'ktev', 'library': 'CD-9940B', 'eod_cookie': '0000_000000000_0000679', 'current_location': '0000_000000000_0000011', 'user_inhibit': ['none', 'none'], 'sum_rd_access': 79, 'system_inhibit': ['none', 'migrating'], 'external_label': 'VO6332', 'wrapper': 'cern', 'remaining_bytes': 1683470848L, 'sum_mounts': 51, 'capacity_bytes': 214748364800L, 'media_type': '9940B', 'last_access': 1257400825.0, 'status': ('ok', None), 'si_time': [0.0, 1257400122.0], 'non_del_files': 226, 'sum_wr_err': 0, 'sum_wr_access': 226, 'file_family': 'ktev_migration', 'address': ('131.225.13.59', 7502), 'volume_family': 'ktev.ktev_migration.cern', 'modification_time': 1177350851.0, 'write_protected': 'y', 'sum_rd_err': 0, 'first_access': 1088397667.0}, 'outfilepath': '/data/data1/Migration_Spool/VO6332:0000_000000000_0000011', 'encp': {'agetime': 30, 'adminpri': 6, 'delpri': 1, 'basepri': 1000007, 'delayed_dismount': 2, 'curpri': 1000007}, 'file_size': 1891545456L, 'mover': {'max_buffer': 1048576000L, 'check_written_file': 10, 'max_failures': 5, 'library': 'CD-9940B.library_manager', 'local_mover': 0, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7575, 'mc_device': '0,0,10,15', 'compression': 0, 'serial_num': '479002012194', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 28311552.0, 'vendor_id': 'STK', 'driver': 'FTTDriver', 'host': 'stkenmvr10a', 'max_time_in_state': 1200, 'device': 'stkenmvr10a:/dev/rmt/tps2d0n', 'name': '9940B10.mover', 'send_stats': 1, 'product_id': 'T9940B', 'statistics_path': '/tmp/enstore/enstore/DBT10MV.stat', 'mover_address': ('131.225.13.37', 7575), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.37', 36806), 'data_ip': 'stkenmvr10a', 'logname': 'DBT10MV', 'hostip': '131.225.13.37', 'do_cleaning': 'No', 'media_changer': 'stk.media_changer'}, 'lm': {'address': ('131.225.13.4', 7522)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKE0/QKE090/f0', 'uid': 0, 'uname': 'root', 'rmajor': 0, 'machine': ('Linux', 'stkendm7a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (33188, 525017360L, 20L, 1, 5406, 2539, 1891545456L, 1088398805, 1088398805, 1088367178), 'mode_octal': '0100744', 'mode': 33252, 'size_bytes': 1891545456L, 'gname': 'root', 'gid': 0, 'fullname': '/data/data1/Migration_Spool/VO6332:0000_000000000_0000011', 'inode': 525017360L, 'minor': 20}, 'version': 'v3_8  CVS $Revision: 1.963 $ encp.pyc', 'encp_daq': None, 'client_crc': 1, 'infile': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKE0/QKE090/.(access)(001F000000000000014B2110)', 'status': ('ok', None), 'volume': 'VO6332', 'outfile': '/data/data1/Migration_Spool/VO6332:0000_000000000_0000011', 'fc': {'status': ('ok', None), 'pnfsid': '001F000000000000014B2110', 'pnfs_name0': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKE0/QKE090/f0', 'deleted': 'no', 'external_label': 'VO6332', 'drive': 'stkenmvr23a:/dev/rmt/tps0d0n:4790000200', 'update': None, 'gid': 0, 'location_cookie': '0000_000000000_0000011', 'complete_crc': 2899242434L, 'address': ('131.225.13.59', 7501), 'uid': 0, 'bfid': 'CDMS108839880300000', 'sanity_cookie': (65536L, 115691675L), 'size': 1891545456L}, 'infilepath': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKE0/QKE090/f0', 'at_the_top': 1792, 'bfid': 'CDMS108839880300000', 'local_inode': 39503171L, 'resend': {'max_retry': 3, 'resubmits': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.19', 50802), 'work': 'read_from_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257400491.869041, 'seek_time': 0.0065901279449462891, 'transfer_time': 61.929592847824097, 't0': 1257400491, 'mount_time': 0.020583868026733398, 'lm_dequeued': 1257400825.3903811, 'job_queued': 1257400492.035696, 'drive_call_time': 51.623392820358276, 'in_queue': 333.35422801971436, 'drive_transfer_time': 59.34771871566772(trunc. 4160) Thread net_thread
00:01:27 stkenmvr10a.fnal.gov 005910 root I DBT10MV  Done is sent Thread net_thread
00:01:27 stkenmvr10a.fnal.gov 005910 root I DBT10MV  maybe_clean Thread net_thread
00:01:27 stkendm7a.fnal.gov 031675 root I ENCP  Received final dialog for stkendm7a.fnal.gov-1257400491-31675-3.  elapsed=764.510sec Thread READ
00:01:27 stkendm7a.fnal.gov 031675 root I ENCP  leaving transfer_file(). Thread READ
00:01:27 stkenmvr10a.fnal.gov 005910 root I DBT10MV  returned from maybe_clean Thread net_thread
00:01:27 stkendm7a.fnal.gov 031675 root I ENCP  MSG_TYPE=ENCP_XFER    read_from_hsm /pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKE0/QKE090/f0 -> /data/data1/Migration_Spool/VO6332:0000_000000000_0000011: 1891545456 bytes copied from VO6332 at 29.1 MB/S (39.5 MB/S network) (30.4 MB/S drive) (62.5 MB/S disk) mover=9940B10.mover drive_id=T9940B drive_sn=479002012194 drive_vendor=STK elapsed=395.63 {'storage_group': 'ktev', 'encp_crc': 2899242434L, 'disk_rate': 62.467440563333803, 'drive_rate': 30.39574829058958, 'driver': 'FTTDriver', 'transfer_rate': 29.116962938691287, 'mover_interface': 'stkenmvr10a', 'overall_rate': 29.072884958580683, 'media_changer': 'stk.media_changer', 'network_rate': 29.116962938691287, 'encp_ip': '131.225.13.19', 'unique_id': 'stkendm7a.fnal.gov-1257400491-31675-3'} Thread READ
00:01:27 stkendm7a.fnal.gov 031675 root I ENCP  File /data/data1/Migration_Spool/VO6332:0000_000000000_0000011 copied successfully. Thread READ
00:01:27 stkendm7a.fnal.gov 031675 root I ENCP  Completed transferring 1891545456 bytes in 1 files in 395.642858982 sec.  Overall rate = 29.1 MB/sec.  Transfer rate = 29.1 MB/sec.  Network rate = 39.5 MB/sec.  Drive rate = 30.4 MB/sec.  Disk rate = 62.5 MB/sec.  Exit status = 0. Thread READ
00:01:27 stkendm7a.fnal.gov 031675 root I ENCP  Version: v3_8  CVS $Revision: 1.963 $ encp.pyc  OS: Linux 2.6.9-89.0.9.ELsmp i686 Release:  Scientific Linux Fermi LTS release 4.4 (Wilson), User: root(0)  Group: root(0)  Euser: root(0)  Egroup: root(0)  Current working directory: stkendm7a.fnal.gov:/opt/enstore/src  Command line: encp --delayed-dismount 2 --ignore-fair-share --bypass-filesystem-max-filesize-check --priority 0 --threaded /pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKCC/QKCC10/f15 /data/data1/Migration_Spool/VO6332:0000_000000000_0000020 Thread READ
00:01:27 stkendm7a.fnal.gov 031675 root I ENCP  Creating zero length output files. Thread READ
00:01:27 stkendm7a.fnal.gov 031675 root I ENCP  Sending /pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKCC/QKCC10/f15 read request to LM: unique_id: stkendm7a.fnal.gov-1257400887-31675-6 inputfile: /pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKCC/QKCC10/.(access)(001F000000000000014AE750) outputfile: /data/data1/Migration_Spool/VO6332:0000_000000000_0000020 Thread READ
00:01:27 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:01:27 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  MSG_TYPE=ADD_TO_LMQ  request added to pending queue for storage group : ktev
00:01:27 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  read Q'd /pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKCC/QKCC10/f15 (stkendm7a.fnal.gov-1257400887-31675-6) -> /data/data1/Migration_Spool/VO6332:0000_000000000_0000020 : library=CD-9940B family=ktev.ktev_migration.cern requester:root
00:01:28 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:01:28 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  write_to_hsm next work on vol=VON262 mover=LTO4_109.mover requester:root
00:01:28 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  HAVE_BOUND:sending write_to_hsm /pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/713/n13047134_0023_L010185R_D07_r4.cand.dogwood1.0.root to mover LTO4_109.mover ('131.225.13.205', 7869) DEL_DISM None
00:01:28 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  MSG_TYPE=ADD_TO_LMQ  request added to work at movers queue for storage group : minos
00:01:28 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  HAVE_BOUND:sending {'vc': {'storage_group': 'minos', 'external_label': 'VON262', 'library': 'CD-LTO4G1', 'file_family': 'minos', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'minos.minos.cpio_odc', 'file_family_width': 1}, 'outfilepath': '/pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/713/n13047134_0023_L010185R_D07_r4.cand.dogwood1.0.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 395831683L, 'mover': 'LTO4_109.mover', 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/713/n13047134_0023_L010185R_D07_r4.cand.dogwood1.0.root', 'uid': 0, 'uname': 'root', 'type': 'cpio_odc', 'gid': 0, 'mtime': 1257400886, 'rmajor': 0, 'machine': ('Linux', 'stkendca21a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (16893, 254031912L, 20L, 1, 42411, 5111, 512L, 1257400834, 1257400834, 1257395375), 'mode_octal': '0100036', 'mode': 33188, 'gname': 'root', 'size_bytes': 395831683L, 'fullname': '/diskb/write-pool-2/data/000F0000000000000B247230', 'inode': 254046768L, 'minor': 20}, 'version': 'v3_8b  CVS $Revision: 1.922 $ <frozen>', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskb/write-pool-2/data/000F0000000000000B247230', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/713/.(access)(000F0000000000000B247230)', 'fc': {'size': 395831683L, 'external_label': 'VON262', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/diskb/write-pool-2/data/000F0000000000000B247230', 'at_the_top': 9, 'local_inode': 1073742568L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.21', 44463), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257400888.2821479, 'encp_start_time': 1257400886.173373, 'job_queued': 1257400887.0234311, 't0': 1257400886, 'in_queue': 1.2584388256072998}, 'user_level': 'user2', 'unique_id': 'stkendca21a.fnal.gov-1257400886-1465-0'} to mover
00:01:28 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:01:28 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  WRITE_TO_HSM Thread MainThread
00:01:28 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  SENDING {'vc': {'storage_group': 'minos', 'external_label': 'VON262', 'library': 'CD-LTO4G1', 'file_family': 'minos', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'minos.minos.cpio_odc', 'file_family_width': 1}, 'outfilepath': '/pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/713/n13047134_0023_L010185R_D07_r4.cand.dogwood1.0.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 395831683L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7869, 'mc_device': '0,5,1,7', 'compression': 0, 'serial_num': '9310025229', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr209a', 'max_time_in_state': 1200, 'device': 'stkenmvr209a:/dev/rmt/tps0d0n', 'name': 'LTO4_109.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_109.stat', 'mover_address': ('131.225.13.205', 7869), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.205', 42018), 'data_ip': 'stkenmvr209a', 'logname': 'LTO4_109MV', 'hostip': '131.225.13.205', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/713/n13047134_0023_L010185R_D07_r4.cand.dogwood1.0.root', 'uid': 0, 'uname': 'root', 'mtime': 1257400886, 'rmajor': 0, 'machine': ('Linux', 'stkendca21a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16893, 254031912L, 20L, 1, 42411, 5111, 512L, 1257400834, 1257400834, 1257395375), 'mode_octal': '0100036', 'mode': 33188, 'size_bytes': 395831683L, 'gname': 'root', 'gid': 0, 'fullname': '/diskb/write-pool-2/data/000F0000000000000B247230', 'inode': 254046768L, 'minor': 20}, 'version': 'v3_8b  CVS $Revision: 1.922 $ <frozen>', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskb/write-pool-2/data/000F0000000000000B247230', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/713/.(access)(000F0000000000000B247230)', 'fc': {'size': 395831683L, 'external_label': 'VON262', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/diskb/write-pool-2/data/000F0000000000000B247230', 'at_the_top': 9, 'local_inode': 1073742568L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.21', 44463), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257400888.2821479, 'encp_start_time': 1257400886.173373, 'job_queued': 1257400887.0234311, 't0': 1257400886, 'in_queue': 1.2584388256072998}, 'user_level': 'user2', 'unique_id': 'stkendca21a.fnal.gov-1257400886-1465-0'} Thread client_connect_thread
00:01:28 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:01:28 stkendca21a.fnal.gov 001465 root I ENCP  Control socket ('131.225.13.21', 44463) is connected to ('131.225.13.205', 42019) for stkendca21a.fnal.gov-1257400886-1465-0.  elapsed=2.150sec
00:01:28 stkendca21a.fnal.gov 001465 root I ENCP  Received callback ticket from mover LTO4_109.mover for transfer stkendca21a.fnal.gov-1257400886-1465-0.  elapsed=2.150sec
00:01:28 stkendca21a.fnal.gov 001465 root I ENCP  Atempting to connect data socket to mover at ('131.225.13.205', 42018).
00:01:28 stkendca21a.fnal.gov 001465 root I ENCP  Connecting data socket to mover (('131.225.13.205', 42018)) with interface 131.225.13.21.  elapsed=2.150sec
00:01:28 stkendca21a.fnal.gov 001465 root I ENCP  Data socket ('131.225.13.21', 44465) is connected to ('131.225.13.205', 42018).  elapsed=2.150sec
00:01:28 stkendca21a.fnal.gov 001465 root I ENCP  Opening local file.
00:01:28 stkendca21a.fnal.gov 001465 root I ENCP  pf_status = ('ok', None)  skip_layer_cleanup = False
00:01:28 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  tape position: target 857 current 857 0 Thread seek_thread
00:01:28 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  write_tape starting, bytes_to_write=395832320 Thread tape_thread
00:01:28 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  Write starting. Tape VON262 absolute location in blocks 2534610 Thread tape_thread
00:01:28 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  <storage groups {'ktev': [('9940B35.mover', 'VO6341'), ('9940B10.mover', 'VO6332')]} volume_families {'ktev.ktev_migration.cern': [('9940B35.mover', 'VO6341'), ('9940B10.mover', 'VO6332')]} >
00:01:28 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  read_from_hsm next work on vol=VO6332 mover=9940B10.mover requester:root
00:01:28 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  HAVE_BOUND:sending read_from_hsm /pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKE0/QKE075/f0 to mover 9940B10.mover ('131.225.13.37', 7575) DEL_DISM 2
00:01:28 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  MSG_TYPE=ADD_TO_LMQ  request added to work at movers queue for storage group : ktev
00:01:28 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  HAVE_BOUND:sending {'vc': {'comment': '', 'declared': 1083263100.0, 'blocksize': 131072, 'storage_group': 'ktev', 'library': 'CD-9940B', 'eod_cookie': '0000_000000000_0000679', 'current_location': '0000_000000000_0000014', 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'migrating'], 'external_label': 'VO6332', 'wrapper': 'cern', 'remaining_bytes': 1683470848L, 'sum_mounts': 51, 'sum_rd_access': 78, 'media_type': '9940B', 'last_access': 1257400500.0, 'status': ('ok', None), 'si_time': [0.0, 1257400122.0], 'capacity_bytes': 214748364800L, 'non_del_files': 226, 'sum_wr_err': 0, 'sum_wr_access': 226, 'file_family': 'ktev_migration', 'address': ('131.225.13.59', 7502), 'volume_family': 'ktev.ktev_migration.cern', 'modification_time': 1177350851.0, 'write_protected': 'y', 'sum_rd_err': 0, 'first_access': 1088397667.0}, 'outfilepath': '/data/data1/Migration_Spool/VO6332:0000_000000000_0000014', 'encp': {'agetime': 30, 'adminpri': 6, 'delpri': 1, 'basepri': 1000007, 'delayed_dismount': 2, 'curpri': 1000007}, 'file_size': 9804820352L, 'mover': '9940B10.mover', 'lm': {'address': ('131.225.13.4', 7522)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKE0/QKE075/f0', 'uid': 0, 'uname': 'root', 'gid': 0, 'rmajor': 0, 'machine': ('Linux', 'stkendm7a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (33188, 525006672L, 20L, 1, 5406, 2539, 1L, 1088399251, 1088399251, 1088348242), 'mode_octal': '0100744', 'mode': 33252, 'gname': 'root', 'size_bytes': 9804820352L, 'fullname': '/data/data1/Migration_Spool/VO6332:0000_000000000_0000014', 'inode': 525006672L, 'minor': 20}, 'version': 'v3_8  CVS $Revision: 1.963 $ encp.pyc', 'encp_daq': None, 'client_crc': 1, 'infile': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKE0/QKE075/.(access)(001F000000000000014AF750)', 'status': ('ok', None), 'volume': 'VO6332', 'outfile': '/data/data1/Migration_Spool/VO6332:0000_000000000_0000014', 'fc': {'status': ('ok', None), 'pnfsid': '001F000000000000014AF750', 'pnfs_name0': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKE0/QKE075/f0', 'deleted': 'no', 'external_label': 'VO6332', 'drive': 'stkenmvr23a:/dev/rmt/tps0d0n:4790000200', 'update': None, 'gid': 0, 'location_cookie': '0000_000000000_0000014', 'complete_crc': 747488339L, 'address': ('131.225.13.59', 7501), 'uid': 0, 'bfid': 'CDMS108839925100000', 'sanity_cookie': (65536L, 1434457559L), 'size': 9804820352L}, 'infilepath': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKE0/QKE075/f0', 'at_the_top': 197, 'bfid': 'CDMS108839925100000', 'local_inode': 39503172L, 'resend': {'max_retry': 3, 'resubmits': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.19', 50804), 'work': 'read_from_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257400888.430691, 'encp_start_time': 1257400500.83413, 'job_queued': 1257400500.9833369, 't0': 1257400500, 'in_queue': 387.44689917564392}, 'user_level': 'admin', 'unique_id': 'stkendm7a.fnal.gov-1257400500-31675-4'} to mover
00:01:28 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  <storage groups {'ktev': [('9940B35.mover', 'VO6341'), ('9940B10.mover', 'VO6332')]} volume_families {'ktev.ktev_migration.cern': [('9940B35.mover', 'VO6341'), ('9940B10.mover', 'VO6332')]} >
00:01:28 stkenmvr10a.fnal.gov 005910 root I DBT10MV  READ FROM HSM Thread MainThread
00:01:28 stkendm7a.fnal.gov 031675 root I ENCP  Listening for control socket at: ('131.225.13.19', 50804) Thread READ_1
00:01:28 stkenmvr10a.fnal.gov 005910 root I DBT10MV  SENDING {'vc': {'comment': '', 'declared': 1083263100.0, 'blocksize': 131072, 'storage_group': 'ktev', 'library': 'CD-9940B', 'eod_cookie': '0000_000000000_0000679', 'current_location': '0000_000000000_0000014', 'user_inhibit': ['none', 'none'], 'sum_rd_access': 78, 'system_inhibit': ['none', 'migrating'], 'external_label': 'VO6332', 'wrapper': 'cern', 'remaining_bytes': 1683470848L, 'sum_mounts': 51, 'capacity_bytes': 214748364800L, 'media_type': '9940B', 'last_access': 1257400500.0, 'status': ('ok', None), 'si_time': [0.0, 1257400122.0], 'non_del_files': 226, 'sum_wr_err': 0, 'sum_wr_access': 226, 'file_family': 'ktev_migration', 'address': ('131.225.13.59', 7502), 'volume_family': 'ktev.ktev_migration.cern', 'modification_time': 1177350851.0, 'write_protected': 'y', 'sum_rd_err': 0, 'first_access': 1088397667.0}, 'outfilepath': '/data/data1/Migration_Spool/VO6332:0000_000000000_0000014', 'encp': {'agetime': 30, 'adminpri': 6, 'delpri': 1, 'basepri': 1000007, 'delayed_dismount': 2, 'curpri': 1000007}, 'file_size': 9804820352L, 'mover': {'max_buffer': 1048576000L, 'check_written_file': 10, 'max_failures': 5, 'library': 'CD-9940B.library_manager', 'local_mover': 0, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7575, 'mc_device': '0,0,10,15', 'compression': 0, 'serial_num': '479002012194', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 28311552.0, 'vendor_id': 'STK', 'driver': 'FTTDriver', 'host': 'stkenmvr10a', 'max_time_in_state': 1200, 'device': 'stkenmvr10a:/dev/rmt/tps2d0n', 'name': '9940B10.mover', 'send_stats': 1, 'product_id': 'T9940B', 'statistics_path': '/tmp/enstore/enstore/DBT10MV.stat', 'mover_address': ('131.225.13.37', 7575), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.37', 36809), 'data_ip': 'stkenmvr10a', 'logname': 'DBT10MV', 'hostip': '131.225.13.37', 'do_cleaning': 'No', 'media_changer': 'stk.media_changer'}, 'lm': {'address': ('131.225.13.4', 7522)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKE0/QKE075/f0', 'uid': 0, 'uname': 'root', 'rmajor': 0, 'machine': ('Linux', 'stkendm7a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (33188, 525006672L, 20L, 1, 5406, 2539, 1L, 1088399251, 1088399251, 1088348242), 'mode_octal': '0100744', 'mode': 33252, 'size_bytes': 9804820352L, 'gname': 'root', 'gid': 0, 'fullname': '/data/data1/Migration_Spool/VO6332:0000_000000000_0000014', 'inode': 525006672L, 'minor': 20}, 'version': 'v3_8  CVS $Revision: 1.963 $ encp.pyc', 'encp_daq': None, 'client_crc': 1, 'infile': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKE0/QKE075/.(access)(001F000000000000014AF750)', 'status': ('ok', None), 'volume': 'VO6332', 'outfile': '/data/data1/Migration_Spool/VO6332:0000_000000000_0000014', 'fc': {'status': ('ok', None), 'pnfsid': '001F000000000000014AF750', 'pnfs_name0': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKE0/QKE075/f0', 'deleted': 'no', 'external_label': 'VO6332', 'drive': 'stkenmvr23a:/dev/rmt/tps0d0n:4790000200', 'update': None, 'gid': 0, 'location_cookie': '0000_000000000_0000014', 'complete_crc': 747488339L, 'address': ('131.225.13.59', 7501), 'uid': 0, 'bfid': 'CDMS108839925100000', 'sanity_cookie': (65536L, 1434457559L), 'size': 9804820352L}, 'infilepath': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKE0/QKE075/f0', 'at_the_top': 197, 'bfid': 'CDMS108839925100000', 'local_inode': 39503172L, 'resend': {'max_retry': 3, 'resubmits': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.19', 50804), 'work': 'read_from_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257400888.430691, 'encp_start_time': 1257400500.83413, 'job_queued': 1257400500.9833369, 't0': 1257400500, 'in_queue': 387.44689917564392}, 'user_level': 'admin', 'unique_id': 'stkendm7a.fnal.gov-1257400500-31675-4'} Thread client_connect_thread
00:01:28 stkendm7a.fnal.gov 031675 root I ENCP  Waiting for mover to connect control socket at address: ('131.225.13.19', 50804)  elapsed=765.550sec Thread READ_1
00:01:28 stkendm7a.fnal.gov 031675 root I ENCP  Control socket ('131.225.13.19', 50804) is connected to ('131.225.13.37', 36810) for stkendm7a.fnal.gov-1257400500-31675-4.  elapsed=765.560sec Thread READ_1
00:01:28 stkendm7a.fnal.gov 031675 root I ENCP  Received callback ticket from mover 9940B10.mover for transfer stkendm7a.fnal.gov-1257400500-31675-4.  elapsed=765.560sec Thread READ_1
00:01:28 stkendm7a.fnal.gov 031675 root I ENCP  Attempting to connect data socket to mover at ('131.225.13.37', 36809). Thread READ_1
00:01:28 stkendm7a.fnal.gov 031675 root I ENCP  Connecting data socket to mover (('131.225.13.37', 36809)) with interface 131.225.13.19.  elapsed=765.560sec Thread READ_1
00:01:28 stkendm7a.fnal.gov 031675 root I ENCP  Data socket ('131.225.13.19', 50822) is connected to ('131.225.13.37', 36809).  elapsed=765.560sec Thread READ_1
00:01:28 stkendm7a.fnal.gov 031675 root I ENCP  Opening local file. Thread READ_1
00:01:28 stkenmvr10a.fnal.gov 005910 root I DBT10MV  tape position: target 14 current 11 14432 Thread seek_thread
00:01:28 stkenmvr10a.fnal.gov 005910 root I DBT10MV  skip_fm 3 Thread seek_thread
00:01:28 stkenmvr10a.fnal.gov 005910 root I DBT10MV  read_tape starting, bytes_to_read=9804820352 Thread tape_thread
00:01:29 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  <storage groups {'ktev': [('9940B35.mover', 'VO6341'), ('9940B10.mover', 'VO6332')]} volume_families {'ktev.ktev_migration.cern': [('9940B35.mover', 'VO6341'), ('9940B10.mover', 'VO6332')]} >
00:01:29 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:01:32 last message repeated 1 times
00:01:32 stkendca21a.fnal.gov 001465 root I ENCP  File /diskb/write-pool-2/data/000F0000000000000B247230 transfered for stkendca21a.fnal.gov-1257400886-1465-0 in 4.46241998672 seconds.
00:01:32 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  written bytes 395832320/395832320, blocks 3020 header 213 trailer 424 Thread tape_thread
00:01:33 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:01:34 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  filemarks written. Tape VON262 absolute location in blocks 2537631 Thread tape_thread
00:01:34 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  write_tape timing:pull 0.034243106842 write 0.761422872543 crc_check 1.85180211067 freespace 0.0409190654755 block_write 2.68838715553 idle 0.0621528625488 Thread tape_thread
00:01:34 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  drive stats after write. Tape VON262 position 857 block 0 block_size 0 bloc_loc 2537631 tot_blocks 762939000 BOT 0 read_err 0 write_err 0 bytes 395832320 block_write_tot 2.68838715553 tape_rate 147237840.794 Thread tape_thread
00:01:34 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  remaining bytes info in DB 464807936000 estimated 464412103680 reported from drive 464439296000 position 858 Thread tape_thread
00:01:34 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  new bitfile request {'uid': 0, 'external_label': 'VON262', 'gid': 0, 'location_cookie': '0000_000000000_0000857', 'complete_crc': 4237717571L, 'sanity_cookie': (65536L, 353651973L), 'size': 395831683L} Thread tape_thread
00:01:34 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  set remaining: VON262 464439296000 0000_000000000_0000858 Thread tape_thread
00:01:34 last message repeated 1 times
00:01:34 stkensrv0n.fnal.gov 028762 enstore I VOLSRV  Updating  			update volume set last_access = '2009-11-05 00:01:34', eod_cookie = '0000_000000000_0000858', non_del_files = '860', remaining_bytes = 464439296000 where label = 'VON262';
00:01:34 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  transfer complete volume=VON262 location=858 Thread tape_thread
00:01:34 last message repeated 1 times
00:01:34 stkensrv0n.fnal.gov 028762 enstore I VOLSRV  Updating  			update volume set sum_wr_access = '857', modification_time = '2009-11-05 00:01:34' where label = 'VON262';
00:01:34 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  Sending done to client: {'vc': {'comment': '', 'storage_group': 'minos', 'blocksize': 131072, 'declared': 1248994044.0, 'library': 'CD-LTO4G1', 'si_time': [0.0, 1242939993.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VON262', 'wrapper': 'cpio_odc', 'remaining_bytes': 464807936000L, 'sum_mounts': 156, 'sum_rd_access': 0, 'file_family_width': 1, 'last_access': 1257400833.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0000857', 'capacity_bytes': 858993459200L, 'non_del_files': 859, 'sum_wr_err': 0, 'sum_wr_access': 856, 'file_family': 'minos', 'address': ('131.225.13.59', 7502), 'volume_family': 'minos.minos.cpio_odc', 'modification_time': 1257400833.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1257286976.0}, 'outfilepath': '/pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/713/n13047134_0023_L010185R_D07_r4.cand.dogwood1.0.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 395831683L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7869, 'mc_device': '0,5,1,7', 'compression': 0, 'serial_num': '9310025229', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr209a', 'max_time_in_state': 1200, 'device': 'stkenmvr209a:/dev/rmt/tps0d0n', 'name': 'LTO4_109.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_109.stat', 'mover_address': ('131.225.13.205', 7869), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.205', 42018), 'data_ip': 'stkenmvr209a', 'logname': 'LTO4_109MV', 'hostip': '131.225.13.205', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/713/n13047134_0023_L010185R_D07_r4.cand.dogwood1.0.root', 'uid': 0, 'uname': 'root', 'mtime': 1257400886, 'rmajor': 0, 'machine': ('Linux', 'stkendca21a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16893, 254031912L, 20L, 1, 42411, 5111, 512L, 1257400834, 1257400834, 1257395375), 'mode_octal': '0100036', 'mode': 33188, 'size_bytes': 395831683L, 'gname': 'root', 'gid': 0, 'fullname': '/diskb/write-pool-2/data/000F0000000000000B247230', 'inode': 254046768L, 'minor': 20}, 'version': 'v3_8b  CVS $Revision: 1.922 $ <frozen>', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskb/write-pool-2/data/000F0000000000000B247230', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/713/.(access)(000F0000000000000B247230)', 'fc': {'uid': 0, 'external_label': 'VON262', 'gid': 0, 'location_cookie': '0000_000000000_0000857', 'complete_crc': 4237717571L, 'bfid': 'CDMS125740089400000', 'sanity_cookie': (65536L, 353651973L), 'size': 395831683L}, 'infilepath': '/diskb/write-pool-2/data/000F0000000000000B247230', 'at_the_top': 9, 'local_inode': 1073742568L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.21', 44463), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257400886.173373, 'seek_time': 0.029946088790893555, 'transfer_time': 6.1454980373382568, 't0': 1257400886, 'mount_time': 0.018849849700927734, 'lm_dequeued': 1257400888.2821479, 'job_queued': 1257400887.0234311, 'drive_call_time': 0.71732807159423828, 'in_queue': 1.2584388256072998, 'drive_transfer_time': 4.3167307376861572}, 'user_level': 'user2', 'unique_i(trunc. 4125) Thread tape_thread
00:01:34 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  Done is sent Thread tape_thread
00:01:34 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  maybe_clean Thread tape_thread
00:01:34 stkendca21a.fnal.gov 001465 root I ENCP  Received final dialog for stkendca21a.fnal.gov-1257400886-1465-0.  elapsed=8.390sec
00:01:34 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  returned from maybe_clean Thread tape_thread
00:01:34 stkendca21a.fnal.gov 001465 root I ENCP  leaving transfer_file().
00:01:34 stkendca21a.fnal.gov 001465 root I ENCP  pf_status = ('ok', None)  skip_layer_cleanup = False
00:01:34 stkensrv0n.fnal.gov 028796 enstore I FILSRV  Updating  			update file set deleted = 'n', pnfs_path = '/pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/713/n13047134_0023_L010185R_D07_r4.cand.dogwood1.0.root', drive = 'stkenmvr209a:/dev/rmt/tps0d0n:9310025229', pnfs_id = '000F0000000000000B247230' where bfid = 'CDMS125740089400000';
00:01:34 stkendca21a.fnal.gov 001465 root I ENCP  MSG_TYPE=ENCP_XFER    write_to_hsm /diskb/write-pool-2/data/000F0000000000000B247230 -> /pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/713/n13047134_0023_L010185R_D07_r4.cand.dogwood1.0.root: 395831683 bytes copied to VON262 at 61.2 MB/S (77.7 MB/S network) (87.4 MB/S drive) (84.7 MB/S disk) mover=LTO4_109.mover drive_id=ULTRIUM-TD4 drive_sn=9310025229 drive_vendor=IBM elapsed=8.8026 {'storage_group': 'minos', 'encp_crc': 4237717571L, 'disk_rate': 84.74201902454314, 'drive_rate': 87.449167584472519, 'driver': 'FTTDriver', 'transfer_rate': 61.161297654490511, 'mover_interface': 'stkenmvr209a', 'overall_rate': 56.731251297284906, 'media_changer': 'SL8500G1.media_changer', 'network_rate': 61.161297654490511, 'encp_ip': '131.225.13.21', 'unique_id': 'stkendca21a.fnal.gov-1257400886-1465-0'}
00:01:34 stkendca21a.fnal.gov 001465 root I ENCP  File /pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/713/n13047134_0023_L010185R_D07_r4.cand.dogwood1.0.root copied successfully.
00:01:34 stkendca21a.fnal.gov 001465 root I ENCP  Completed transferring 395831683 bytes in 1 files in 8.8037519455 sec.  Overall rate = 56.7 MB/sec.  Transfer rate = 61.2 MB/sec.  Network rate = 77.7 MB/sec.  Drive rate = 87.4 MB/sec.  Disk rate = 84.7 MB/sec.  Exit status = 0.
00:01:35 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:01:35 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  write_to_hsm next work on vol=VON262 mover=LTO4_109.mover requester:root
00:01:35 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  HAVE_BOUND:sending write_to_hsm /pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/712/n13047129_0017_L010185R_D07_r4.cand.dogwood1.0.root to mover LTO4_109.mover ('131.225.13.205', 7869) DEL_DISM None
00:01:35 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  MSG_TYPE=ADD_TO_LMQ  request added to work at movers queue for storage group : minos
00:01:35 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  HAVE_BOUND:sending {'vc': {'storage_group': 'minos', 'external_label': 'VON262', 'library': 'CD-LTO4G1', 'file_family': 'minos', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'minos.minos.cpio_odc', 'file_family_width': 1}, 'outfilepath': '/pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/712/n13047129_0017_L010185R_D07_r4.cand.dogwood1.0.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 407234060L, 'mover': 'LTO4_109.mover', 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/712/n13047129_0017_L010185R_D07_r4.cand.dogwood1.0.root', 'uid': 0, 'uname': 'root', 'type': 'cpio_odc', 'gid': 0, 'mtime': 1257400886, 'rmajor': 0, 'machine': ('Linux', 'stkendca21a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (16893, 254012896L, 20L, 1, 42411, 5111, 512L, 1257400849, 1257400849, 1257387546), 'mode_octal': '0100036', 'mode': 33188, 'gname': 'root', 'size_bytes': 407234060L, 'fullname': '/diskb/write-pool-2/data/000F0000000000000B247260', 'inode': 254046816L, 'minor': 20}, 'version': 'v3_8b  CVS $Revision: 1.922 $ <frozen>', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskb/write-pool-2/data/000F0000000000000B247260', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/712/.(access)(000F0000000000000B247260)', 'fc': {'size': 407234060L, 'external_label': 'VON262', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/diskb/write-pool-2/data/000F0000000000000B247260', 'at_the_top': 62, 'local_inode': 1073743793L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.21', 44464), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257400895.361366, 'encp_start_time': 1257400886.17538, 'job_queued': 1257400887.063122, 't0': 1257400886, 'in_queue': 8.2981770038604736}, 'user_level': 'user2', 'unique_id': 'stkendca21a.fnal.gov-1257400886-1456-0'} to mover
00:01:35 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:01:35 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  WRITE_TO_HSM Thread MainThread
00:01:35 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  SENDING {'vc': {'storage_group': 'minos', 'external_label': 'VON262', 'library': 'CD-LTO4G1', 'file_family': 'minos', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'minos.minos.cpio_odc', 'file_family_width': 1}, 'outfilepath': '/pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/712/n13047129_0017_L010185R_D07_r4.cand.dogwood1.0.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 407234060L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7869, 'mc_device': '0,5,1,7', 'compression': 0, 'serial_num': '9310025229', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr209a', 'max_time_in_state': 1200, 'device': 'stkenmvr209a:/dev/rmt/tps0d0n', 'name': 'LTO4_109.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_109.stat', 'mover_address': ('131.225.13.205', 7869), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.205', 42020), 'data_ip': 'stkenmvr209a', 'logname': 'LTO4_109MV', 'hostip': '131.225.13.205', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/712/n13047129_0017_L010185R_D07_r4.cand.dogwood1.0.root', 'uid': 0, 'uname': 'root', 'mtime': 1257400886, 'rmajor': 0, 'machine': ('Linux', 'stkendca21a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16893, 254012896L, 20L, 1, 42411, 5111, 512L, 1257400849, 1257400849, 1257387546), 'mode_octal': '0100036', 'mode': 33188, 'size_bytes': 407234060L, 'gname': 'root', 'gid': 0, 'fullname': '/diskb/write-pool-2/data/000F0000000000000B247260', 'inode': 254046816L, 'minor': 20}, 'version': 'v3_8b  CVS $Revision: 1.922 $ <frozen>', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskb/write-pool-2/data/000F0000000000000B247260', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/712/.(access)(000F0000000000000B247260)', 'fc': {'size': 407234060L, 'external_label': 'VON262', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/diskb/write-pool-2/data/000F0000000000000B247260', 'at_the_top': 62, 'local_inode': 1073743793L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.21', 44464), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257400895.361366, 'encp_start_time': 1257400886.17538, 'job_queued': 1257400887.063122, 't0': 1257400886, 'in_queue': 8.2981770038604736}, 'user_level': 'user2', 'unique_id': 'stkendca21a.fnal.gov-1257400886-1456-0'} Thread client_connect_thread
00:01:35 stkendca21a.fnal.gov 001456 root I ENCP  Control socket ('131.225.13.21', 44464) is connected to ('131.225.13.205', 42021) for stkendca21a.fnal.gov-1257400886-1456-0.  elapsed=9.230sec
00:01:35 stkendca21a.fnal.gov 001456 root I ENCP  Received callback ticket from mover LTO4_109.mover for transfer stkendca21a.fnal.gov-1257400886-1456-0.  elapsed=9.230sec
00:01:35 stkendca21a.fnal.gov 001456 root I ENCP  Atempting to connect data socket to mover at ('131.225.13.205', 42020).
00:01:35 stkendca21a.fnal.gov 001456 root I ENCP  Connecting data socket to mover (('131.225.13.205', 42020)) with interface 131.225.13.21.  elapsed=9.230sec
00:01:35 stkendca21a.fnal.gov 001456 root I ENCP  Data socket ('131.225.13.21', 44467) is connected to ('131.225.13.205', 42020).  elapsed=9.230sec
00:01:35 stkendca21a.fnal.gov 001456 root I ENCP  Opening local file.
00:01:35 stkendca21a.fnal.gov 001456 root I ENCP  pf_status = ('ok', None)  skip_layer_cleanup = False
00:01:35 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  tape position: target 858 current 858 0 Thread seek_thread
00:01:35 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  write_tape starting, bytes_to_write=407234560 Thread tape_thread
00:01:35 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  Write starting. Tape VON262 absolute location in blocks 2537631 Thread tape_thread
00:01:35 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:01:36 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:01:37 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:01:40 last message repeated 1 times
00:01:40 stkendca21a.fnal.gov 001456 root I ENCP  File /diskb/write-pool-2/data/000F0000000000000B247260 transfered for stkendca21a.fnal.gov-1257400886-1456-0 in 4.79174208641 seconds.
00:01:40 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  written bytes 407234560/407234560, blocks 3107 header 213 trailer 287 Thread tape_thread
00:01:41 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  filemarks written. Tape VON262 absolute location in blocks 2540739 Thread tape_thread
00:01:41 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  write_tape timing:pull 0.0352473258972 write 0.377767801285 crc_check 1.84670567513 freespace 0.03959608078 block_write 2.29931688309 idle 0.0634968280792 Thread tape_thread
00:01:41 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  drive stats after write. Tape VON262 position 858 block 0 block_size 0 bloc_loc 2540739 tot_blocks 762939000 BOT 0 read_err 0 write_err 0 bytes 407234560 block_write_tot 2.29931688309 tape_rate 177111107.649 Thread tape_thread
00:01:41 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  remaining bytes info in DB 464439296000 estimated 464032061440 reported from drive 464059392000 position 859 Thread tape_thread
00:01:41 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  new bitfile request {'uid': 0, 'external_label': 'VON262', 'gid': 0, 'location_cookie': '0000_000000000_0000858', 'complete_crc': 3657442903L, 'sanity_cookie': (65536L, 2400446363L), 'size': 407234060L} Thread tape_thread
00:01:41 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  set remaining: VON262 464059392000 0000_000000000_0000859 Thread tape_thread
00:01:41 last message repeated 1 times
00:01:41 stkensrv0n.fnal.gov 028762 enstore I VOLSRV  Updating  			update volume set last_access = '2009-11-05 00:01:41', eod_cookie = '0000_000000000_0000859', non_del_files = '861', remaining_bytes = 464059392000 where label = 'VON262';
00:01:41 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  transfer complete volume=VON262 location=859 Thread tape_thread
00:01:41 last message repeated 1 times
00:01:41 stkensrv0n.fnal.gov 028762 enstore I VOLSRV  Updating  			update volume set sum_wr_access = '858', modification_time = '2009-11-05 00:01:41' where label = 'VON262';
00:01:41 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  Sending done to client: {'vc': {'comment': '', 'storage_group': 'minos', 'blocksize': 131072, 'declared': 1248994044.0, 'library': 'CD-LTO4G1', 'si_time': [0.0, 1242939993.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VON262', 'wrapper': 'cpio_odc', 'remaining_bytes': 464439296000L, 'sum_mounts': 156, 'sum_rd_access': 0, 'file_family_width': 1, 'last_access': 1257400894.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0000858', 'capacity_bytes': 858993459200L, 'non_del_files': 860, 'sum_wr_err': 0, 'sum_wr_access': 857, 'file_family': 'minos', 'address': ('131.225.13.59', 7502), 'volume_family': 'minos.minos.cpio_odc', 'modification_time': 1257400894.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1257286976.0}, 'outfilepath': '/pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/712/n13047129_0017_L010185R_D07_r4.cand.dogwood1.0.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 407234060L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7869, 'mc_device': '0,5,1,7', 'compression': 0, 'serial_num': '9310025229', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr209a', 'max_time_in_state': 1200, 'device': 'stkenmvr209a:/dev/rmt/tps0d0n', 'name': 'LTO4_109.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_109.stat', 'mover_address': ('131.225.13.205', 7869), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.205', 42020), 'data_ip': 'stkenmvr209a', 'logname': 'LTO4_109MV', 'hostip': '131.225.13.205', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/712/n13047129_0017_L010185R_D07_r4.cand.dogwood1.0.root', 'uid': 0, 'uname': 'root', 'mtime': 1257400886, 'rmajor': 0, 'machine': ('Linux', 'stkendca21a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16893, 254012896L, 20L, 1, 42411, 5111, 512L, 1257400849, 1257400849, 1257387546), 'mode_octal': '0100036', 'mode': 33188, 'size_bytes': 407234060L, 'gname': 'root', 'gid': 0, 'fullname': '/diskb/write-pool-2/data/000F0000000000000B247260', 'inode': 254046816L, 'minor': 20}, 'version': 'v3_8b  CVS $Revision: 1.922 $ <frozen>', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskb/write-pool-2/data/000F0000000000000B247260', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/712/.(access)(000F0000000000000B247260)', 'fc': {'uid': 0, 'external_label': 'VON262', 'gid': 0, 'location_cookie': '0000_000000000_0000858', 'complete_crc': 3657442903L, 'bfid': 'CDMS125740090100000', 'sanity_cookie': (65536L, 2400446363L), 'size': 407234060L}, 'infilepath': '/diskb/write-pool-2/data/000F0000000000000B247260', 'at_the_top': 62, 'local_inode': 1073743793L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.21', 44464), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257400886.17538, 'seek_time': 0.03060603141784668, 'transfer_time': 6.1731340885162354, 't0': 1257400886, 'mount_time': 0.024980068206787109, 'lm_dequeued': 1257400895.361366, 'job_queued': 1257400887.063122, 'drive_call_time': 0.33277201652526855, 'in_queue': 8.2981770038604736, 'drive_transfer_time': 3.589646577835083}, 'user_level': 'user2', 'unique_id':(trunc. 4122) Thread tape_thread
00:01:41 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  Done is sent Thread tape_thread
00:01:41 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  maybe_clean Thread tape_thread
00:01:41 stkendca21a.fnal.gov 001456 root I ENCP  Received final dialog for stkendca21a.fnal.gov-1257400886-1456-0.  elapsed=15.490sec
00:01:41 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  returned from maybe_clean Thread tape_thread
00:01:41 stkendca21a.fnal.gov 001456 root I ENCP  leaving transfer_file().
00:01:41 stkendca21a.fnal.gov 001456 root I ENCP  pf_status = ('ok', None)  skip_layer_cleanup = False
00:01:41 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:01:41 stkensrv0n.fnal.gov 028796 enstore I FILSRV  Updating  			update file set deleted = 'n', pnfs_path = '/pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/712/n13047129_0017_L010185R_D07_r4.cand.dogwood1.0.root', drive = 'stkenmvr209a:/dev/rmt/tps0d0n:9310025229', pnfs_id = '000F0000000000000B247260' where bfid = 'CDMS125740090100000';
00:01:41 stkendca21a.fnal.gov 001456 root I ENCP  MSG_TYPE=ENCP_XFER    write_to_hsm /diskb/write-pool-2/data/000F0000000000000B247260 -> /pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/712/n13047129_0017_L010185R_D07_r4.cand.dogwood1.0.root: 407234060 bytes copied to VON262 at 62.7 MB/S (74.7 MB/S network) (108 MB/S drive) (81.2 MB/S disk) mover=LTO4_109.mover drive_id=ULTRIUM-TD4 drive_sn=9310025229 drive_vendor=IBM elapsed=15.81 {'storage_group': 'minos', 'encp_crc': 3657442903L, 'disk_rate': 81.158127091106664, 'drive_rate': 108.19133732716023, 'driver': 'FTTDriver', 'transfer_rate': 62.736964652980333, 'mover_interface': 'stkenmvr209a', 'overall_rate': 58.984809603957444, 'media_changer': 'SL8500G1.media_changer', 'network_rate': 62.736964652980333, 'encp_ip': '131.225.13.21', 'unique_id': 'stkendca21a.fnal.gov-1257400886-1456-0'}
00:01:41 stkendca21a.fnal.gov 001456 root I ENCP  File /pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/712/n13047129_0017_L010185R_D07_r4.cand.dogwood1.0.root copied successfully.
00:01:41 stkendca21a.fnal.gov 001456 root I ENCP  Completed transferring 407234060 bytes in 1 files in 15.810710907 sec.  Overall rate = 59 MB/sec.  Transfer rate = 62.7 MB/sec.  Network rate = 74.7 MB/sec.  Drive rate = 108 MB/sec.  Disk rate = 81.2 MB/sec.  Exit status = 0.
00:01:42 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:01:43 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:01:44 stkendca17a.fnal.gov 030753 root I ENCP  File /diska/write-pool-1/data/0030000000000000061EB100 transfered for stkendca17a.fnal.gov-1257400813-30753-0 in 36.3089258671 seconds.
00:01:44 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  <storage groups {'ktev': [('9940B10.mover', 'VO6332'), ('9940B35.mover', 'VO6341')]} volume_families {'ktev.ktev_migration.cern': [('9940B10.mover', 'VO6332'), ('9940B35.mover', 'VO6341')]} >
00:01:44 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:01:44 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV  skip_fm -1288 Thread seek_thread
00:01:44 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV  skip_fm 1 Thread seek_thread
00:01:44 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV  read_tape starting, bytes_to_read=2118453701 Thread tape_thread
00:01:45 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  <storage groups {'ktev': [('9940B35.mover', 'VO6341'), ('9940B10.mover', 'VO6332')]} volume_families {'ktev.ktev_migration.cern': [('9940B35.mover', 'VO6341'), ('9940B10.mover', 'VO6332')]} >
00:01:45 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:01:46 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:01:47 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  written bytes 1966080512/1966080512, blocks 15001 header 189 trailer 323 Thread tape_thread
00:01:47 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:01:48 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:01:49 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:01:50 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:01:51 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  filemarks written. Tape VON743 absolute location in blocks 1923462 Thread tape_thread
00:01:51 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  write_tape timing:pull 0.169672012329 write 9.55598902702 crc_check 7.75009202957 freespace 0.159265518188 block_write 17.6350185871 idle 0.275107622147 Thread tape_thread
00:01:51 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  drive stats after write. Tape VON743 position 132 block 0 block_size 0 bloc_loc 1923462 tot_blocks 762939000 BOT 0 read_err 0 write_err 0 bytes 1966080512 block_write_tot 17.6350185871 tape_rate 111487294.572 Thread tape_thread
00:01:51 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  remaining bytes info in DB 542455808000 estimated 540489727488 reported from drive 538925056000 position 133 Thread tape_thread
00:01:51 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  new bitfile request {'uid': 0, 'external_label': 'VON743', 'gid': 0, 'location_cookie': '0000_000000000_0000132', 'complete_crc': 4275116931L, 'sanity_cookie': (65536L, 195262742L), 'size': 1966080000L} Thread tape_thread
00:01:51 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:01:51 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  set remaining: VON743 538925056000 0000_000000000_0000133 Thread tape_thread
00:01:51 last message repeated 1 times
00:01:51 stkensrv0n.fnal.gov 028762 enstore I VOLSRV  Updating  			update volume set last_access = '2009-11-05 00:01:51', eod_cookie = '0000_000000000_0000133', non_del_files = '134', remaining_bytes = 538925056000 where label = 'VON743';
00:01:51 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  transfer complete volume=VON743 location=133 Thread tape_thread
00:01:52 last message repeated 1 times
00:01:52 stkensrv0n.fnal.gov 028762 enstore I VOLSRV  Updating  			update volume set sum_wr_access = '132', modification_time = '2009-11-05 00:01:51' where label = 'VON743';
00:01:52 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  Sending done to client: {'vc': {'comment': '', 'storage_group': 'exp-db', 'blocksize': 131072, 'declared': 1256853181.0, 'library': 'CD-LTO4G1', 'si_time': [0.0, 1255497368.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VON743', 'wrapper': 'cpio_odc', 'remaining_bytes': 542455808000L, 'sum_mounts': 46, 'sum_rd_access': 0, 'file_family_width': 2, 'last_access': 1257400867.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0000132', 'capacity_bytes': 858993459200L, 'non_del_files': 133, 'sum_wr_err': 0, 'sum_wr_access': 131, 'file_family': 'daily-d0-offline', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.daily-d0-offline.cpio_odc', 'modification_time': 1257400867.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1257340831.0}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702070562_s103266_p22', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 1966080000L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'dismount_delay': 30, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7842, 'mc_device': '0,7,1,12', 'compression': 0, 'serial_num': '1310155269', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr142a', 'max_time_in_state': 1200, 'device': 'stkenmvr142a:/dev/rmt/tps0d0n', 'name': 'LTO4_42.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_42.stat', 'single_filemark': 1, 'mount_delay': 15, 'mover_address': ('131.225.13.170', 7842), 'check_first_written_file': 0, 'callback_addr': ('131.225.13.170', 52227), 'data_ip': 'stkenmvr142a', 'logname': 'LTO4_42MV', 'hostip': '131.225.13.170', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer', 'min_buffer': 67108864L}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702070562_s103266_p22', 'uid': 0, 'uname': 'root', 'mtime': 1257400813, 'rmajor': 0, 'machine': ('Linux', 'stkendca17a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 820619016L, 20L, 1, 1602, 2752, 512L, 1257400777, 1257400777, 1254398418), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 1966080000L, 'gname': 'root', 'gid': 0, 'fullname': '/diska/write-pool-1/data/0030000000000000061EB100', 'inode': 807317760L, 'minor': 20}, 'version': 'v3_8b  CVS $Revision: 1.922 $ <frozen>', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diska/write-pool-1/data/0030000000000000061EB100', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/.(access)(0030000000000000061EB100)', 'fc': {'uid': 0, 'external_label': 'VON743', 'gid': 0, 'location_cookie': '0000_000000000_0000132', 'complete_crc': 4275116931L, 'bfid': 'CDMS125740091100000', 'sanity_cookie': (65536L, 195262742L), 'size': 1966080000L}, 'infilepath': '/diska/write-pool-1/data/0030000000000000061EB100', 'at_the_top': 281, 'local_inode': 268435937L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.5', 44347), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257400812.029377, 'seek_time': 0.033883810043334961, 'transfer_time': 44.173907041549683, 't0': 1257400812, 'mount_time': 0.042756080627441406, 'lm_dequeued': 1257400867.6867249, 'job_queued': 1257400814.1018741, 'drive_call_time': 9.3641271591186523, 'in_queue': 53.5845627784729, 'drive_transfer_time': 22.161067008972168}, 'user_level': 'us(trunc. 4141) Thread tape_thread
00:01:52 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  Done is sent Thread tape_thread
00:01:52 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  maybe_clean Thread tape_thread
00:01:52 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  returned from maybe_clean Thread tape_thread
00:01:52 stkendca17a.fnal.gov 030753 root I ENCP  Received final dialog for stkendca17a.fnal.gov-1257400813-30753-0.  elapsed=100.010sec
00:01:52 stkendca17a.fnal.gov 030753 root I ENCP  leaving transfer_file().
00:01:52 stkendca17a.fnal.gov 030753 root I ENCP  pf_status = ('ok', None)  skip_layer_cleanup = False
00:01:52 stkensrv0n.fnal.gov 028796 enstore I FILSRV  Updating  			update file set deleted = 'n', pnfs_path = '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702070562_s103266_p22', drive = 'stkenmvr142a:/dev/rmt/tps0d0n:1310155269', pnfs_id = '0030000000000000061EB100' where bfid = 'CDMS125740091100000';
00:01:52 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:01:52 stkendca17a.fnal.gov 030753 root I ENCP  MSG_TYPE=ENCP_XFER    write_to_hsm /diska/write-pool-1/data/0030000000000000061EB100 -> /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702070562_s103266_p22: 1966080000 bytes copied to VON743 at 42.4 MB/S (79.2 MB/S network) (84.6 MB/S drive) (51.7 MB/S disk) mover=LTO4_42.mover drive_id=ULTRIUM-TD4 drive_sn=1310155269 drive_vendor=IBM elapsed=100.34 {'storage_group': 'exp-db', 'encp_crc': 4275116931L, 'disk_rate': 51.674704127143826, 'drive_rate': 84.607839470946246, 'driver': 'FTTDriver', 'transfer_rate': 42.426604407978282, 'mover_interface': 'stkenmvr142a', 'overall_rate': 42.015910602670417, 'media_changer': 'SL8500G1.media_changer', 'network_rate': 42.426604407978282, 'encp_ip': '131.225.13.5', 'unique_id': 'stkendca17a.fnal.gov-1257400813-30753-0'}
00:01:52 stkendca17a.fnal.gov 030753 root I ENCP  File /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702070562_s103266_p22 copied successfully.
00:01:52 stkendca17a.fnal.gov 030753 root I ENCP  Completed transferring 1966080000 bytes in 1 files in 100.343653917 sec.  Overall rate = 42 MB/sec.  Transfer rate = 42.4 MB/sec.  Network rate = 79.2 MB/sec.  Drive rate = 84.6 MB/sec.  Disk rate = 51.7 MB/sec.  Exit status = 0.
00:01:52 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:01:52 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  write_to_hsm next work on vol=VON743 mover=LTO4_42.mover requester:root
00:01:52 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  HAVE_BOUND:sending write_to_hsm /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702070562_s103266_p25 to mover LTO4_42.mover ('131.225.13.170', 7842) DEL_DISM None
00:01:52 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  MSG_TYPE=ADD_TO_LMQ  request added to work at movers queue for storage group : exp-db
00:01:52 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  HAVE_BOUND:sending {'vc': {'storage_group': 'exp-db', 'external_label': 'VON743', 'library': 'CD-LTO4G1', 'file_family': 'daily-d0-offline', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.daily-d0-offline.cpio_odc', 'file_family_width': 2}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702070562_s103266_p25', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 1654611968L, 'mover': 'LTO4_42.mover', 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702070562_s103266_p25', 'uid': 0, 'uname': 'root', 'type': 'cpio_odc', 'gid': 0, 'mtime': 1257400876, 'rmajor': 0, 'machine': ('Linux', 'stkendca18a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (16877, 820619016L, 20L, 1, 1602, 2752, 512L, 1257400848, 1257400848, 1254398418), 'mode_octal': '0100016', 'mode': 33188, 'gname': 'root', 'size_bytes': 1654611968L, 'fullname': '/diska/write-pool-1/data/0030000000000000061EB170', 'inode': 807317872L, 'minor': 20}, 'version': 'v3_8b  CVS $Revision: 1.922 $ <frozen>', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diska/write-pool-1/data/0030000000000000061EB170', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/.(access)(0030000000000000061EB170)', 'fc': {'size': 1654611968L, 'external_label': 'VON743', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/diska/write-pool-1/data/0030000000000000061EB170', 'at_the_top': 339, 'local_inode': 622464081L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.6', 54459), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257400912.722043, 'encp_start_time': 1257400874.723093, 'job_queued': 1257400876.4799681, 't0': 1257400874, 'in_queue': 36.241796970367432}, 'user_level': 'user2', 'unique_id': 'stkendca18a.fnal.gov-1257400876-21252-0'} to mover
00:01:52 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:01:52 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  WRITE_TO_HSM Thread MainThread
00:01:52 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  SENDING {'vc': {'storage_group': 'exp-db', 'external_label': 'VON743', 'library': 'CD-LTO4G1', 'file_family': 'daily-d0-offline', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.daily-d0-offline.cpio_odc', 'file_family_width': 2}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702070562_s103266_p25', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 1654611968L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'dismount_delay': 30, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7842, 'mc_device': '0,7,1,12', 'compression': 0, 'serial_num': '1310155269', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr142a', 'max_time_in_state': 1200, 'device': 'stkenmvr142a:/dev/rmt/tps0d0n', 'name': 'LTO4_42.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_42.stat', 'single_filemark': 1, 'mount_delay': 15, 'mover_address': ('131.225.13.170', 7842), 'check_first_written_file': 0, 'callback_addr': ('131.225.13.170', 52230), 'data_ip': 'stkenmvr142a', 'logname': 'LTO4_42MV', 'hostip': '131.225.13.170', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer', 'min_buffer': 67108864L}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702070562_s103266_p25', 'uid': 0, 'uname': 'root', 'mtime': 1257400876, 'rmajor': 0, 'machine': ('Linux', 'stkendca18a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 820619016L, 20L, 1, 1602, 2752, 512L, 1257400848, 1257400848, 1254398418), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 1654611968L, 'gname': 'root', 'gid': 0, 'fullname': '/diska/write-pool-1/data/0030000000000000061EB170', 'inode': 807317872L, 'minor': 20}, 'version': 'v3_8b  CVS $Revision: 1.922 $ <frozen>', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diska/write-pool-1/data/0030000000000000061EB170', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/.(access)(0030000000000000061EB170)', 'fc': {'size': 1654611968L, 'external_label': 'VON743', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/diska/write-pool-1/data/0030000000000000061EB170', 'at_the_top': 339, 'local_inode': 622464081L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.6', 54459), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257400912.722043, 'encp_start_time': 1257400874.723093, 'job_queued': 1257400876.4799681, 't0': 1257400874, 'in_queue': 36.241796970367432}, 'user_level': 'user2', 'unique_id': 'stkendca18a.fnal.gov-1257400876-21252-0'} Thread client_connect_thread
00:01:52 stkendca18a.fnal.gov 021252 root I ENCP  Control socket ('131.225.13.6', 54459) is connected to ('131.225.13.170', 52231) for stkendca18a.fnal.gov-1257400876-21252-0.  elapsed=38.060sec
00:01:52 stkendca18a.fnal.gov 021252 root I ENCP  Received callback ticket from mover LTO4_42.mover for transfer stkendca18a.fnal.gov-1257400876-21252-0.  elapsed=38.070sec
00:01:52 stkendca18a.fnal.gov 021252 root I ENCP  Atempting to connect data socket to mover at ('131.225.13.170', 52230).
00:01:52 stkendca18a.fnal.gov 021252 root I ENCP  Connecting data socket to mover (('131.225.13.170', 52230)) with interface 131.225.13.6.  elapsed=38.070sec
00:01:52 stkendca18a.fnal.gov 021252 root I ENCP  Data socket ('131.225.13.6', 54463) is connected to ('131.225.13.170', 52230).  elapsed=38.070sec
00:01:52 stkendca18a.fnal.gov 021252 root I ENCP  Opening local file.
00:01:52 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  tape position: target 133 current 133 0 Thread seek_thread
00:01:52 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  write_tape starting, bytes_to_write=1654612480 Thread tape_thread
00:01:52 stkendca18a.fnal.gov 021252 root I ENCP  pf_status = ('ok', None)  skip_layer_cleanup = False
00:01:52 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  Write starting. Tape VON743 absolute location in blocks 1923462 Thread tape_thread
00:01:53 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:01:53 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:01:54 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:01:54 stkenmvr35a.fnal.gov 005964 root I DBT35MV  read bytes 11055727536/11055727536, blocks 84349 header 0 Thread tape_thread
00:01:54 stkenmvr35a.fnal.gov 005964 root I DBT35MV  write_client: wrote 11055727536/11055727536 bytes Thread net_thread
00:01:54 stkenmvr35a.fnal.gov 005964 root I DBT35MV  write_client waits for tape thread to exit Thread net_thread
00:01:55 stkenmvr35a.fnal.gov 005964 root I DBT35MV  drive stats after read. Tape VO6341 position 17 block 84349 block_size 0 bloc_loc 355662 tot_blocks 0 BOT 0 read_err 0 write_err 0 bytes 11055727536 block_read_tot 354.344805717 tape_rate 31200478.6231 Thread tape_thread
00:01:55 stkenmvr35a.fnal.gov 005964 root I DBT35MV  read_tape exiting, read 11055727536/11055727536 bytes Thread tape_thread
00:01:55 stkenmvr35a.fnal.gov 005964 root I DBT35MV  read_tape timing:get_space 1.07871818542 read 315.758951187 crc_check 36.4094092846 push 1.09772706032 block_read 354.344805717 idle 2.97483205795 Thread tape_thread
00:01:55 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:01:55 stkenmvr35a.fnal.gov 005964 root I DBT35MV  write_client exits Thread net_thread
00:01:56 stkenmvr35a.fnal.gov 005964 root I DBT35MV  transfer complete volume=VO6341 location=17 Thread net_thread
00:01:56 last message repeated 1 times
00:01:56 stkensrv0n.fnal.gov 028762 enstore I VOLSRV  Updating  			update volume set sum_rd_access = '94', last_access = '2009-11-05 00:01:56' where label = 'VO6341';
00:01:56 stkenmvr35a.fnal.gov 005964 root I DBT35MV  Sending done to client: {'vc': {'comment': '', 'declared': 1083263105.0, 'blocksize': 131072, 'storage_group': 'ktev', 'library': 'CD-9940B', 'eod_cookie': '0000_000000000_0000172', 'current_location': '0000_000000000_0000017', 'user_inhibit': ['none', 'none'], 'sum_rd_access': 93, 'system_inhibit': ['none', 'migrating'], 'external_label': 'VO6341', 'wrapper': 'cern', 'remaining_bytes': 6809693696L, 'sum_mounts': 54, 'capacity_bytes': 214748364800L, 'media_type': '9940B', 'last_access': 1257400550.0, 'status': ('ok', None), 'si_time': [0.0, 1257399311.0], 'non_del_files': 57, 'sum_wr_err': 0, 'sum_wr_access': 57, 'file_family': 'ktev_migration', 'address': ('131.225.13.59', 7502), 'volume_family': 'ktev.ktev_migration.cern', 'modification_time': 1193243360.0, 'write_protected': 'y', 'sum_rd_err': 0, 'first_access': 1088544231.0}, 'outfilepath': '/data/data2/Migration_Spool/VO6341:0000_000000000_0000017', 'encp': {'agetime': 30, 'adminpri': 6, 'delpri': 1, 'basepri': 1000007, 'delayed_dismount': 2, 'curpri': 1000007}, 'file_size': 11055727536L, 'mover': {'max_buffer': 1048576000L, 'check_written_file': 10, 'max_failures': 5, 'library': 'CD-9940B.library_manager', 'local_mover': 0, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7835, 'mc_device': '0,0,10,1', 'compression': 0, 'serial_num': '479001001549', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 28311552.0, 'vendor_id': 'STK', 'driver': 'FTTDriver', 'host': 'stkenmvr35a', 'max_time_in_state': 1200, 'device': 'stkenmvr35a:/dev/rmt/tps2d0n', 'name': '9940B35.mover', 'send_stats': 1, 'product_id': 'T9940B', 'statistics_path': '/tmp/enstore/enstore/DBT35MV.stat', 'mover_address': ('131.225.13.87', 7835), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.87', 33875), 'data_ip': 'stkenmvr35a', 'logname': 'DBT35MV', 'hostip': '131.225.13.87', 'do_cleaning': 'No', 'media_changer': 'stk.media_changer'}, 'lm': {'address': ('131.225.13.4', 7522)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKE1/QKE144/f1', 'uid': 0, 'uname': 'root', 'rmajor': 0, 'machine': ('Linux', 'stkendm7a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (33188, 525141352L, 20L, 1, 5406, 2539, 1L, 1088548053, 1088548053, 1088543658), 'mode_octal': '0100744', 'mode': 33252, 'size_bytes': 11055727536L, 'gname': 'root', 'gid': 0, 'fullname': '/data/data2/Migration_Spool/VO6341:0000_000000000_0000017', 'inode': 525141352L, 'minor': 20}, 'version': 'v3_8  CVS $Revision: 1.963 $ encp.pyc', 'encp_daq': None, 'client_crc': 1, 'infile': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKE1/QKE144/.(access)(001F000000000000014D0568)', 'status': ('ok', None), 'volume': 'VO6341', 'outfile': '/data/data2/Migration_Spool/VO6341:0000_000000000_0000017', 'fc': {'status': ('ok', None), 'pnfsid': '001F000000000000014D0568', 'pnfs_name0': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKE1/QKE144/f1', 'deleted': 'no', 'external_label': 'VO6341', 'drive': 'stkenmvr10a:/dev/rmt/tps0d0n:4790000020', 'update': None, 'gid': 0, 'location_cookie': '0000_000000000_0000017', 'complete_crc': 2382972001L, 'address': ('131.225.13.59', 7501), 'uid': 0, 'bfid': 'CDMS108854805200001', 'sanity_cookie': (65536L, 2720812553L), 'size': 11055727536L}, 'infilepath': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKE1/QKE144/f1', 'at_the_top': 1251, 'bfid': 'CDMS108854805200001', 'local_inode': 4472839L, 'resend': {'max_retry': 3, 'resubmits': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.19', 50709), 'work': 'read_from_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257399821.2345569, 'seek_time': 0.0062549114227294922, 'transfer_time': 364.62315201759338, 't0': 1257399821, 'mount_time': 0.018301963806152344, 'lm_dequeued': 1257400551.308197, 'job_queued': 1257399821.368082, 'drive_call_time': 313.84655857086182, 'in_queue': 729.9396538734436, 'drive_transfer_time': 355.62091779708862}, 'user_(trunc. 4150) Thread net_thread
00:01:56 stkenmvr35a.fnal.gov 005964 root I DBT35MV  Done is sent Thread net_thread
00:01:56 stkenmvr35a.fnal.gov 005964 root I DBT35MV  maybe_clean Thread net_thread
00:01:56 stkenmvr35a.fnal.gov 005964 root I DBT35MV  returned from maybe_clean Thread net_thread
00:01:56 stkendca18a.fnal.gov 020159 root I ENCP  File /diskc/write-pool-3/data/0030000000000000061EB0D8 transfered for stkendca18a.fnal.gov-1257400814-20159-0 in 28.9406700134 seconds.
00:01:56 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:01:56 stkendm7a.fnal.gov 029514 root I ENCP  File /pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKE1/QKE144/f1 transfered for stkendm7a.fnal.gov-1257399821-29514-5 in 365.069668055 seconds. Thread READ_2
00:01:56 stkendm7a.fnal.gov 029514 root I ENCP  Received final dialog for stkendm7a.fnal.gov-1257399821-29514-5.  elapsed=1604.230sec Thread READ_2
00:01:56 stkendm7a.fnal.gov 029514 root I ENCP  leaving transfer_file(). Thread READ_2
00:01:56 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  <storage groups {'ktev': [('9940B10.mover', 'VO6332'), ('9940B35.mover', 'VO6341')]} volume_families {'ktev.ktev_migration.cern': [('9940B10.mover', 'VO6332'), ('9940B35.mover', 'VO6341')]} >
00:01:56 stkendm7a.fnal.gov 029514 root I ENCP  MSG_TYPE=ENCP_XFER    read_from_hsm /pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKE1/QKE144/f1 -> /data/data2/Migration_Spool/VO6341:0000_000000000_0000017: 11055727536 bytes copied from VO6341 at 28.9 MB/S (39.6 MB/S network) (29.6 MB/S drive) (60.3 MB/S disk) mover=9940B35.mover drive_id=T9940B drive_sn=479001001549 drive_vendor=STK elapsed=1095.3 {'storage_group': 'ktev', 'encp_crc': 2382972001L, 'disk_rate': 60.299979546245254, 'drive_rate': 29.648321773593015, 'driver': 'FTTDriver', 'transfer_rate': 28.880090900236521, 'mover_interface': 'stkenmvr35a', 'overall_rate': 28.870287449690455, 'media_changer': 'stk.media_changer', 'network_rate': 28.880090900236521, 'encp_ip': '131.225.13.19', 'unique_id': 'stkendm7a.fnal.gov-1257399821-29514-5'} Thread READ_2
00:01:56 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  read_from_hsm next work on vol=VO6341 mover=9940B35.mover requester:root
00:01:56 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  HAVE_BOUND:sending read_from_hsm /pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC083/f2 to mover 9940B35.mover ('131.225.13.87', 7835) DEL_DISM 2
00:01:56 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  MSG_TYPE=ADD_TO_LMQ  request added to work at movers queue for storage group : ktev
00:01:56 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  HAVE_BOUND:sending {'vc': {'comment': '', 'declared': 1083263105.0, 'blocksize': 131072, 'storage_group': 'ktev', 'library': 'CD-9940B', 'eod_cookie': '0000_000000000_0000172', 'current_location': '0000_000000000_0000020', 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'migrating'], 'external_label': 'VO6341', 'wrapper': 'cern', 'remaining_bytes': 6809693696L, 'sum_mounts': 54, 'sum_rd_access': 92, 'media_type': '9940B', 'last_access': 1257400185.0, 'status': ('ok', None), 'si_time': [0.0, 1257399311.0], 'capacity_bytes': 214748364800L, 'non_del_files': 57, 'sum_wr_err': 0, 'sum_wr_access': 57, 'file_family': 'ktev_migration', 'address': ('131.225.13.59', 7502), 'volume_family': 'ktev.ktev_migration.cern', 'modification_time': 1193243360.0, 'write_protected': 'y', 'sum_rd_err': 0, 'first_access': 1088544231.0}, 'outfilepath': '/data/data2/Migration_Spool/VO6341:0000_000000000_0000020', 'encp': {'agetime': 30, 'adminpri': 6, 'delpri': 1, 'basepri': 1000007, 'delayed_dismount': 2, 'curpri': 1000007}, 'file_size': 17300L, 'mover': '9940B35.mover', 'lm': {'address': ('131.225.13.4', 7522)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC083/f2', 'uid': 0, 'uname': 'root', 'gid': 0, 'rmajor': 0, 'machine': ('Linux', 'stkendm7a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (33188, 525142272L, 20L, 1, 5406, 2539, 17300L, 1177386516, 1088548216, 1088547491), 'mode_octal': '0100744', 'mode': 33252, 'gname': 'root', 'size_bytes': 17300L, 'fullname': '/data/data2/Migration_Spool/VO6341:0000_000000000_0000020', 'inode': 525142272L, 'minor': 20}, 'version': 'v3_8  CVS $Revision: 1.963 $ encp.pyc', 'encp_daq': None, 'client_crc': 1, 'infile': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC083/.(access)(001F000000000000014D0900)', 'status': ('ok', None), 'volume': 'VO6341', 'outfile': '/data/data2/Migration_Spool/VO6341:0000_000000000_0000020', 'fc': {'status': ('ok', None), 'pnfsid': '001F000000000000014D0900', 'pnfs_name0': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC083/f2', 'deleted': 'no', 'external_label': 'VO6341', 'drive': 'stkenmvr10a:/dev/rmt/tps0d0n:4790000020', 'update': None, 'gid': 0, 'location_cookie': '0000_000000000_0000020', 'complete_crc': 4291618937L, 'address': ('131.225.13.59', 7501), 'uid': 0, 'bfid': 'CDMS108854821600000', 'sanity_cookie': (17300L, 4291618937L), 'size': 17300L}, 'infilepath': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC083/f2', 'at_the_top': 1386, 'bfid': 'CDMS108854821600000', 'local_inode': 4472840L, 'resend': {'max_retry': 3, 'resubmits': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.19', 50794), 'work': 'read_from_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257400916.582844, 'encp_start_time': 1257400186.5711391, 'job_queued': 1257400186.8585629, 't0': 1257400186, 'in_queue': 729.72353100776672}, 'user_level': 'admin', 'unique_id': 'stkendm7a.fnal.gov-1257400186-29514-6'} to mover
00:01:56 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  <storage groups {'ktev': [('9940B10.mover', 'VO6332'), ('9940B35.mover', 'VO6341')]} volume_families {'ktev.ktev_migration.cern': [('9940B10.mover', 'VO6332'), ('9940B35.mover', 'VO6341')]} >
00:01:56 stkendm7a.fnal.gov 029514 root I ENCP  File /data/data2/Migration_Spool/VO6341:0000_000000000_0000017 copied successfully. Thread READ_2
00:01:56 stkendm7a.fnal.gov 029514 root I ENCP  Completed transferring 11055727536 bytes in 1 files in 1095.35072899 sec.  Overall rate = 28.9 MB/sec.  Transfer rate = 28.9 MB/sec.  Network rate = 39.6 MB/sec.  Drive rate = 29.6 MB/sec.  Disk rate = 60.3 MB/sec.  Exit status = 0. Thread READ_2
00:01:56 stkenmvr35a.fnal.gov 005964 root I DBT35MV  READ FROM HSM Thread MainThread
00:01:56 stkendm7a.fnal.gov 029514 root I MIGRATION  Listening for control socket at: ('131.225.13.19', 50794) Thread READ
00:01:56 stkendm7a.fnal.gov 029514 root I MIGRATION  Waiting for mover to connect control socket at address: ('131.225.13.19', 50794)  elapsed=1604.380sec Thread READ
00:01:56 stkenmvr35a.fnal.gov 005964 root I DBT35MV  SENDING {'vc': {'comment': '', 'declared': 1083263105.0, 'blocksize': 131072, 'storage_group': 'ktev', 'library': 'CD-9940B', 'eod_cookie': '0000_000000000_0000172', 'current_location': '0000_000000000_0000020', 'user_inhibit': ['none', 'none'], 'sum_rd_access': 92, 'system_inhibit': ['none', 'migrating'], 'external_label': 'VO6341', 'wrapper': 'cern', 'remaining_bytes': 6809693696L, 'sum_mounts': 54, 'capacity_bytes': 214748364800L, 'media_type': '9940B', 'last_access': 1257400185.0, 'status': ('ok', None), 'si_time': [0.0, 1257399311.0], 'non_del_files': 57, 'sum_wr_err': 0, 'sum_wr_access': 57, 'file_family': 'ktev_migration', 'address': ('131.225.13.59', 7502), 'volume_family': 'ktev.ktev_migration.cern', 'modification_time': 1193243360.0, 'write_protected': 'y', 'sum_rd_err': 0, 'first_access': 1088544231.0}, 'outfilepath': '/data/data2/Migration_Spool/VO6341:0000_000000000_0000020', 'encp': {'agetime': 30, 'adminpri': 6, 'delpri': 1, 'basepri': 1000007, 'delayed_dismount': 2, 'curpri': 1000007}, 'file_size': 17300L, 'mover': {'max_buffer': 1048576000L, 'check_written_file': 10, 'max_failures': 5, 'library': 'CD-9940B.library_manager', 'local_mover': 0, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7835, 'mc_device': '0,0,10,1', 'compression': 0, 'serial_num': '479001001549', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 28311552.0, 'vendor_id': 'STK', 'driver': 'FTTDriver', 'host': 'stkenmvr35a', 'max_time_in_state': 1200, 'device': 'stkenmvr35a:/dev/rmt/tps2d0n', 'name': '9940B35.mover', 'send_stats': 1, 'product_id': 'T9940B', 'statistics_path': '/tmp/enstore/enstore/DBT35MV.stat', 'mover_address': ('131.225.13.87', 7835), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.87', 33886), 'data_ip': 'stkenmvr35a', 'logname': 'DBT35MV', 'hostip': '131.225.13.87', 'do_cleaning': 'No', 'media_changer': 'stk.media_changer'}, 'lm': {'address': ('131.225.13.4', 7522)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC083/f2', 'uid': 0, 'uname': 'root', 'rmajor': 0, 'machine': ('Linux', 'stkendm7a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (33188, 525142272L, 20L, 1, 5406, 2539, 17300L, 1177386516, 1088548216, 1088547491), 'mode_octal': '0100744', 'mode': 33252, 'size_bytes': 17300L, 'gname': 'root', 'gid': 0, 'fullname': '/data/data2/Migration_Spool/VO6341:0000_000000000_0000020', 'inode': 525142272L, 'minor': 20}, 'version': 'v3_8  CVS $Revision: 1.963 $ encp.pyc', 'encp_daq': None, 'client_crc': 1, 'infile': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC083/.(access)(001F000000000000014D0900)', 'status': ('ok', None), 'volume': 'VO6341', 'outfile': '/data/data2/Migration_Spool/VO6341:0000_000000000_0000020', 'fc': {'status': ('ok', None), 'pnfsid': '001F000000000000014D0900', 'pnfs_name0': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC083/f2', 'deleted': 'no', 'external_label': 'VO6341', 'drive': 'stkenmvr10a:/dev/rmt/tps0d0n:4790000020', 'update': None, 'gid': 0, 'location_cookie': '0000_000000000_0000020', 'complete_crc': 4291618937L, 'address': ('131.225.13.59', 7501), 'uid': 0, 'bfid': 'CDMS108854821600000', 'sanity_cookie': (17300L, 4291618937L), 'size': 17300L}, 'infilepath': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC083/f2', 'at_the_top': 1386, 'bfid': 'CDMS108854821600000', 'local_inode': 4472840L, 'resend': {'max_retry': 3, 'resubmits': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.19', 50794), 'work': 'read_from_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257400916.582844, 'encp_start_time': 1257400186.5711391, 'job_queued': 1257400186.8585629, 't0': 1257400186, 'in_queue': 729.72353100776672}, 'user_level': 'admin', 'unique_id': 'stkendm7a.fnal.gov-1257400186-29514-6'} Thread client_connect_thread
00:01:56 stkendm7a.fnal.gov 029514 root I MIGRATION  Control socket ('131.225.13.19', 50794) is connected to ('131.225.13.87', 33887) for stkendm7a.fnal.gov-1257400186-29514-6.  elapsed=1604.380sec Thread READ
00:01:56 stkendm7a.fnal.gov 029514 root I MIGRATION  Received callback ticket from mover 9940B35.mover for transfer stkendm7a.fnal.gov-1257400186-29514-6.  elapsed=1604.390sec Thread READ
00:01:56 stkendm7a.fnal.gov 029514 root I MIGRATION  Attempting to connect data socket to mover at ('131.225.13.87', 33886). Thread READ
00:01:56 stkendm7a.fnal.gov 029514 root I MIGRATION  Connecting data socket to mover (('131.225.13.87', 33886)) with interface 131.225.13.19.  elapsed=1604.390sec Thread READ
00:01:56 stkendm7a.fnal.gov 029514 root I MIGRATION  Data socket ('131.225.13.19', 50823) is connected to ('131.225.13.87', 33886).  elapsed=1604.390sec Thread READ
00:01:56 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  written bytes 1966080512/1966080512, blocks 15001 header 189 trailer 323 Thread tape_thread
00:01:56 stkendm7a.fnal.gov 029514 root I MIGRATION  Opening local file. Thread READ
00:01:56 stkenmvr35a.fnal.gov 005964 root I DBT35MV  tape position: target 20 current 17 84349 Thread seek_thread
00:01:56 stkenmvr35a.fnal.gov 005964 root I DBT35MV  skip_fm 3 Thread seek_thread
00:01:56 stkenmvr35a.fnal.gov 005964 root I DBT35MV  read_tape starting, bytes_to_read=17300 Thread tape_thread
00:01:56 stkenmvr35a.fnal.gov 005964 root I DBT35MV  read bytes 17300/17300, blocks 1 header 0 Thread tape_thread
00:01:56 stkenmvr35a.fnal.gov 005964 root I DBT35MV  write_client: wrote 17300/17300 bytes Thread net_thread
00:01:56 stkenmvr35a.fnal.gov 005964 root I DBT35MV  write_client waits for tape thread to exit Thread net_thread
00:01:56 stkendm7a.fnal.gov 029514 root I MIGRATION  File /pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC083/f2 transfered for stkendm7a.fnal.gov-1257400186-29514-6 in 0.0043249130249 seconds. Thread READ
00:01:56 stkenmvr35a.fnal.gov 005964 root I DBT35MV  drive stats after read. Tape VO6341 position 20 block 1 block_size 0 bloc_loc 355668 tot_blocks 0 BOT 0 read_err 0 write_err 0 bytes 17300 block_read_tot 0.00252795219421 tape_rate 6843483.8442 Thread tape_thread
00:01:56 stkenmvr35a.fnal.gov 005964 root I DBT35MV  read_tape exiting, read 17300/17300 bytes Thread tape_thread
00:01:56 stkenmvr35a.fnal.gov 005964 root I DBT35MV  read_tape timing:get_space 0.000118970870972 read 0.00221180915833 crc_check 0.000179052352905 push 1.81198120117e-05 block_read 0.00252795219421 idle 3.81469726562e-05 Thread tape_thread
00:01:56 stkendm7a.fnal.gov 029514 root I ENCP  Version: v3_8  CVS $Revision: 1.963 $ encp.pyc  OS: Linux 2.6.9-89.0.9.ELsmp i686 Release:  Scientific Linux Fermi LTS release 4.4 (Wilson), User: root(0)  Group: root(0)  Euser: root(0)  Egroup: root(0)  Current working directory: stkendm7a.fnal.gov:/opt/enstore/src  Command line: encp --delayed-dismount 2 --ignore-fair-share --bypass-filesystem-max-filesize-check --priority 0 --threaded /pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC084/f2 /data/data2/Migration_Spool/VO6341:0000_000000000_0000026 Thread READ_2
00:01:56 stkendm7a.fnal.gov 029514 root I ENCP  Creating zero length output files. Thread READ_2
00:01:56 stkendm7a.fnal.gov 029514 root I ENCP  Sending /pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC084/f2 read request to LM: unique_id: stkendm7a.fnal.gov-1257400916-29514-8 inputfile: /pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC084/.(access)(001F000000000000014D0A40) outputfile: /data/data2/Migration_Spool/VO6341:0000_000000000_0000026 Thread READ_2
00:01:56 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  MSG_TYPE=ADD_TO_LMQ  request added to pending queue for storage group : ktev
00:01:56 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  read Q'd /pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC084/f2 (stkendm7a.fnal.gov-1257400916-29514-8) -> /data/data2/Migration_Spool/VO6341:0000_000000000_0000026 : library=CD-9940B family=ktev.ktev_migration.cern requester:root
00:01:57 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  <storage groups {'ktev': [('9940B10.mover', 'VO6332'), ('9940B35.mover', 'VO6341')]} volume_families {'ktev.ktev_migration.cern': [('9940B10.mover', 'VO6332'), ('9940B35.mover', 'VO6341')]} >
00:01:57 stkenmvr35a.fnal.gov 005964 root I DBT35MV  write_client exits Thread net_thread
00:01:57 stkenmvr35a.fnal.gov 005964 root I DBT35MV  transfer complete volume=VO6341 location=20 Thread net_thread
00:01:57 last message repeated 1 times
00:01:57 stkensrv0n.fnal.gov 028762 enstore I VOLSRV  Updating  			update volume set sum_rd_access = '95', last_access = '2009-11-05 00:01:57' where label = 'VO6341';
00:01:57 stkenmvr35a.fnal.gov 005964 root I DBT35MV  Sending done to client: {'vc': {'comment': '', 'declared': 1083263105.0, 'blocksize': 131072, 'storage_group': 'ktev', 'library': 'CD-9940B', 'eod_cookie': '0000_000000000_0000172', 'current_location': '0000_000000000_0000020', 'user_inhibit': ['none', 'none'], 'sum_rd_access': 94, 'system_inhibit': ['none', 'migrating'], 'external_label': 'VO6341', 'wrapper': 'cern', 'remaining_bytes': 6809693696L, 'sum_mounts': 54, 'capacity_bytes': 214748364800L, 'media_type': '9940B', 'last_access': 1257400916.0, 'status': ('ok', None), 'si_time': [0.0, 1257399311.0], 'non_del_files': 57, 'sum_wr_err': 0, 'sum_wr_access': 57, 'file_family': 'ktev_migration', 'address': ('131.225.13.59', 7502), 'volume_family': 'ktev.ktev_migration.cern', 'modification_time': 1193243360.0, 'write_protected': 'y', 'sum_rd_err': 0, 'first_access': 1088544231.0}, 'outfilepath': '/data/data2/Migration_Spool/VO6341:0000_000000000_0000020', 'encp': {'agetime': 30, 'adminpri': 6, 'delpri': 1, 'basepri': 1000007, 'delayed_dismount': 2, 'curpri': 1000007}, 'file_size': 17300L, 'mover': {'max_buffer': 1048576000L, 'check_written_file': 10, 'max_failures': 5, 'library': 'CD-9940B.library_manager', 'local_mover': 0, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7835, 'mc_device': '0,0,10,1', 'compression': 0, 'serial_num': '479001001549', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 28311552.0, 'vendor_id': 'STK', 'driver': 'FTTDriver', 'host': 'stkenmvr35a', 'max_time_in_state': 1200, 'device': 'stkenmvr35a:/dev/rmt/tps2d0n', 'name': '9940B35.mover', 'send_stats': 1, 'product_id': 'T9940B', 'statistics_path': '/tmp/enstore/enstore/DBT35MV.stat', 'mover_address': ('131.225.13.87', 7835), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.87', 33886), 'data_ip': 'stkenmvr35a', 'logname': 'DBT35MV', 'hostip': '131.225.13.87', 'do_cleaning': 'No', 'media_changer': 'stk.media_changer'}, 'lm': {'address': ('131.225.13.4', 7522)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC083/f2', 'uid': 0, 'uname': 'root', 'rmajor': 0, 'machine': ('Linux', 'stkendm7a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (33188, 525142272L, 20L, 1, 5406, 2539, 17300L, 1177386516, 1088548216, 1088547491), 'mode_octal': '0100744', 'mode': 33252, 'size_bytes': 17300L, 'gname': 'root', 'gid': 0, 'fullname': '/data/data2/Migration_Spool/VO6341:0000_000000000_0000020', 'inode': 525142272L, 'minor': 20}, 'version': 'v3_8  CVS $Revision: 1.963 $ encp.pyc', 'encp_daq': None, 'client_crc': 1, 'infile': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC083/.(access)(001F000000000000014D0900)', 'status': ('ok', None), 'volume': 'VO6341', 'outfile': '/data/data2/Migration_Spool/VO6341:0000_000000000_0000020', 'fc': {'status': ('ok', None), 'pnfsid': '001F000000000000014D0900', 'pnfs_name0': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC083/f2', 'deleted': 'no', 'external_label': 'VO6341', 'drive': 'stkenmvr10a:/dev/rmt/tps0d0n:4790000020', 'update': None, 'gid': 0, 'location_cookie': '0000_000000000_0000020', 'complete_crc': 4291618937L, 'address': ('131.225.13.59', 7501), 'uid': 0, 'bfid': 'CDMS108854821600000', 'sanity_cookie': (17300L, 4291618937L), 'size': 17300L}, 'infilepath': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC083/f2', 'at_the_top': 1386, 'bfid': 'CDMS108854821600000', 'local_inode': 4472840L, 'resend': {'max_retry': 3, 'resubmits': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.19', 50794), 'work': 'read_from_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257400186.5711391, 'seek_time': 0.0062692165374755859, 'transfer_time': 1.0100510120391846, 't0': 1257400186, 'mount_time': 0.020855903625488281, 'lm_dequeued': 1257400916.582844, 'job_queued': 1257400186.8585629, 'drive_call_time': 0.0021860599517822266, 'in_queue': 729.72353100776672, 'drive_transfer_time': 0.0025429725646972656}, 'user_level'(trunc. 4144) Thread net_thread
00:01:57 stkenmvr35a.fnal.gov 005964 root I DBT35MV  Done is sent Thread net_thread
00:01:57 stkenmvr35a.fnal.gov 005964 root I DBT35MV  maybe_clean Thread net_thread
00:01:57 stkendm7a.fnal.gov 029514 root I ENCP  Received final dialog for stkendm7a.fnal.gov-1257400186-29514-6.  elapsed=1605.450sec Thread READ
00:01:57 stkenmvr35a.fnal.gov 005964 root I DBT35MV  returned from maybe_clean Thread net_thread
00:01:57 stkendm7a.fnal.gov 029514 root I ENCP  leaving transfer_file(). Thread READ
00:01:57 stkendm7a.fnal.gov 029514 root I ENCP  MSG_TYPE=ENCP_XFER    read_from_hsm /pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC083/f2 -> /data/data2/Migration_Spool/VO6341:0000_000000000_0000020: 17300 bytes copied from VO6341 at 0.0159 MB/S (199 MB/S network) (6.49 MB/S drive) (4.1 MB/S disk) mover=9940B35.mover drive_id=T9940B drive_sn=479001001549 drive_vendor=STK elapsed=731.23 {'storage_group': 'ktev', 'encp_crc': 4291618937L, 'disk_rate': 4.0969868291397127, 'drive_rate': 6.4879054940933809, 'driver': 'FTTDriver', 'transfer_rate': 0.015855511382630378, 'mover_interface': 'stkenmvr35a', 'overall_rate': 0.014259929248495546, 'media_changer': 'stk.media_changer', 'network_rate': 0.015855511382630378, 'encp_ip': '131.225.13.19', 'unique_id': 'stkendm7a.fnal.gov-1257400186-29514-6'} Thread READ
00:01:57 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:01:57 stkendm7a.fnal.gov 029514 root I ENCP  File /data/data2/Migration_Spool/VO6341:0000_000000000_0000020 copied successfully. Thread READ
00:01:57 stkendm7a.fnal.gov 029514 root I ENCP  Completed transferring 17300 bytes in 1 files in 731.237879992 sec.  Overall rate = 0.0143 MB/sec.  Transfer rate = 0.0159 MB/sec.  Network rate = 199 MB/sec.  Drive rate = 6.49 MB/sec.  Disk rate = 4.1 MB/sec.  Exit status = 0. Thread READ
00:01:57 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  filemarks written. Tape VON266 absolute location in blocks 2634427 Thread tape_thread
00:01:57 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  write_tape timing:pull 0.165503501892 write 3.56106567383 crc_check 9.18363237381 freespace 0.192684173584 block_write 13.1028857231 idle 0.307440280914 Thread tape_thread
00:01:57 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  drive stats after write. Tape VON266 position 186 block 0 block_size 0 bloc_loc 2634427 tot_blocks 762939000 BOT 0 read_err 0 write_err 0 bytes 1966080512 block_write_tot 13.1028857231 tape_rate 150049428.313 Thread tape_thread
00:01:57 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  remaining bytes info in DB 455795712000 estimated 453829631488 reported from drive 452280320000 position 187 Thread tape_thread
00:01:57 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  new bitfile request {'uid': 0, 'external_label': 'VON266', 'gid': 0, 'location_cookie': '0000_000000000_0000186', 'complete_crc': 4195779471L, 'sanity_cookie': (65536L, 246509384L), 'size': 1966080000L} Thread tape_thread
00:01:57 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  set remaining: VON266 452280320000 0000_000000000_0000187 Thread tape_thread
00:01:57 last message repeated 1 times
00:01:57 stkensrv0n.fnal.gov 028762 enstore I VOLSRV  Updating  			update volume set last_access = '2009-11-05 00:01:57', eod_cookie = '0000_000000000_0000187', non_del_files = '187', remaining_bytes = 452280320000 where label = 'VON266';
00:01:58 stkendm7a.fnal.gov 029514 root I ENCP  Version: v3_8  CVS $Revision: 1.963 $ encp.pyc  OS: Linux 2.6.9-89.0.9.ELsmp i686 Release:  Scientific Linux Fermi LTS release 4.4 (Wilson), User: root(0)  Group: root(0)  Euser: root(0)  Egroup: root(0)  Current working directory: stkendm7a.fnal.gov:/opt/enstore/src  Command line: encp --delayed-dismount 2 --ignore-fair-share --bypass-filesystem-max-filesize-check --priority 0 --threaded /pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC084/f0 /data/data2/Migration_Spool/VO6341:0000_000000000_0000029 Thread READ
00:01:58 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  transfer complete volume=VON266 location=187 Thread tape_thread
00:01:58 last message repeated 1 times
00:01:58 stkensrv0n.fnal.gov 028762 enstore I VOLSRV  Updating  			update volume set sum_wr_access = '186', last_access = '2009-11-05 00:01:58', modification_time = '2009-11-05 00:01:58' where label = 'VON266';
00:01:58 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  Sending done to client: {'vc': {'comment': '', 'storage_group': 'exp-db', 'blocksize': 131072, 'declared': 1248994048.0, 'library': 'CD-LTO4G1', 'si_time': [0.0, 1242744483.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VON266', 'wrapper': 'cpio_odc', 'remaining_bytes': 455795712000L, 'sum_mounts': 73, 'sum_rd_access': 0, 'file_family_width': 2, 'last_access': 1257400887.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0000186', 'capacity_bytes': 858993459200L, 'non_del_files': 186, 'sum_wr_err': 0, 'sum_wr_access': 185, 'file_family': 'daily-d0-offline', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.daily-d0-offline.cpio_odc', 'modification_time': 1257400887.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1257365852.0}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702068213_s103263_p28', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 1966080000L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7865, 'mc_device': '0,6,1,1', 'compression': 0, 'serial_num': '9310801322', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr205a', 'max_time_in_state': 1200, 'device': 'stkenmvr205a:/dev/rmt/tps0d0n', 'name': 'LTO4_105.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_105.stat', 'mover_address': ('131.225.13.201', 7865), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.201', 42200), 'data_ip': 'stkenmvr205a', 'logname': 'LTO4_105MV', 'hostip': '131.225.13.201', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702068213_s103263_p28', 'uid': 0, 'uname': 'root', 'mtime': 1257400814, 'rmajor': 0, 'machine': ('Linux', 'stkendca18a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 820618800L, 20L, 1, 1602, 2752, 512L, 1257400772, 1257400772, 1254398418), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 1966080000L, 'gname': 'root', 'gid': 0, 'fullname': '/diskc/write-pool-3/data/0030000000000000061EB0D8', 'inode': 807317720L, 'minor': 20}, 'version': 'v3_8b  CVS $Revision: 1.922 $ <frozen>', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskc/write-pool-3/data/0030000000000000061EB0D8', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/.(access)(0030000000000000061EB0D8)', 'fc': {'uid': 0, 'external_label': 'VON266', 'gid': 0, 'location_cookie': '0000_000000000_0000186', 'complete_crc': 4195779471L, 'bfid': 'CDMS125740091700000', 'sanity_cookie': (65536L, 246509384L), 'size': 1966080000L}, 'infilepath': '/diskc/write-pool-3/data/0030000000000000061EB0D8', 'at_the_top': 302, 'local_inode': 536876407L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.6', 54453), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257400813.491024, 'seek_time': 0.029940128326416016, 'transfer_time': 30.723655939102173, 't0': 1257400813, 'mount_time': 0.017549037933349609, 'lm_dequeued': 1257400887.20081, 'job_queued': 1257400814.430378, 'drive_call_time': 3.3439593315124512, 'in_queue': 72.770142078399658, 'drive_transfer_time': 14.872918844223022}, 'user_level': 'user2', 'unique_id': 'stkendca18a.fnal.gov-1257400814-20159-0'} Thread tape_thread
00:01:58 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  Done is sent Thread tape_thread
00:01:58 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  maybe_clean Thread tape_thread
00:01:58 stkendm7a.fnal.gov 029514 root I ENCP  Creating zero length output files. Thread READ
00:01:58 stkendm7a.fnal.gov 029514 root I ENCP  Sending /pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC084/f0 read request to LM: unique_id: stkendm7a.fnal.gov-1257400918-29514-9 inputfile: /pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC084/.(access)(001F000000000000014D0980) outputfile: /data/data2/Migration_Spool/VO6341:0000_000000000_0000029 Thread READ
00:01:58 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  returned from maybe_clean Thread tape_thread
00:01:58 stkendca18a.fnal.gov 020159 root I ENCP  Received final dialog for stkendca18a.fnal.gov-1257400814-20159-0.  elapsed=104.590sec
00:01:58 stkendca18a.fnal.gov 020159 root I ENCP  leaving transfer_file().
00:01:58 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  MSG_TYPE=ADD_TO_LMQ  request added to pending queue for storage group : ktev
00:01:58 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  read Q'd /pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC084/f0 (stkendm7a.fnal.gov-1257400918-29514-9) -> /data/data2/Migration_Spool/VO6341:0000_000000000_0000029 : library=CD-9940B family=ktev.ktev_migration.cern requester:root
00:01:58 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:01:58 stkendca18a.fnal.gov 020159 root I ENCP  pf_status = ('ok', None)  skip_layer_cleanup = False
00:01:58 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  write_to_hsm next work on vol=VON266 mover=LTO4_105.mover requester:root
00:01:58 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  HAVE_BOUND:sending write_to_hsm /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702075233_s103280_p2 to mover LTO4_105.mover ('131.225.13.201', 7865) DEL_DISM None
00:01:58 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  MSG_TYPE=ADD_TO_LMQ  request added to work at movers queue for storage group : exp-db
00:01:58 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  HAVE_BOUND:sending {'vc': {'storage_group': 'exp-db', 'external_label': 'VON266', 'library': 'CD-LTO4G1', 'file_family': 'daily-d0-offline', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.daily-d0-offline.cpio_odc', 'file_family_width': 2}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702075233_s103280_p2', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 1966080000L, 'mover': 'LTO4_105.mover', 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702075233_s103280_p2', 'uid': 0, 'uname': 'root', 'type': 'cpio_odc', 'gid': 0, 'mtime': 1257400876, 'rmajor': 0, 'machine': ('Linux', 'stkendca18a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (16877, 820618800L, 20L, 1, 1602, 2752, 512L, 1257400847, 1257400847, 1254398418), 'mode_octal': '0100016', 'mode': 33188, 'gname': 'root', 'size_bytes': 1966080000L, 'fullname': '/diska/write-pool-1/data/0030000000000000061EB148', 'inode': 807317832L, 'minor': 20}, 'version': 'v3_8b  CVS $Revision: 1.922 $ <frozen>', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diska/write-pool-1/data/0030000000000000061EB148', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/.(access)(0030000000000000061EB148)', 'fc': {'size': 1966080000L, 'external_label': 'VON266', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/diska/write-pool-1/data/0030000000000000061EB148', 'at_the_top': 86, 'local_inode': 536955558L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.6', 54460), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257400918.22574, 'encp_start_time': 1257400874.730144, 'job_queued': 1257400876.62868, 't0': 1257400874, 'in_queue': 41.597016096115112}, 'user_level': 'user2', 'unique_id': 'stkendca18a.fnal.gov-1257400876-21257-0'} to mover
00:01:58 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:01:58 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  WRITE_TO_HSM Thread MainThread
00:01:58 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  SENDING {'vc': {'storage_group': 'exp-db', 'external_label': 'VON266', 'library': 'CD-LTO4G1', 'file_family': 'daily-d0-offline', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.daily-d0-offline.cpio_odc', 'file_family_width': 2}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702075233_s103280_p2', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 1966080000L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7865, 'mc_device': '0,6,1,1', 'compression': 0, 'serial_num': '9310801322', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr205a', 'max_time_in_state': 1200, 'device': 'stkenmvr205a:/dev/rmt/tps0d0n', 'name': 'LTO4_105.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_105.stat', 'mover_address': ('131.225.13.201', 7865), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.201', 42203), 'data_ip': 'stkenmvr205a', 'logname': 'LTO4_105MV', 'hostip': '131.225.13.201', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702075233_s103280_p2', 'uid': 0, 'uname': 'root', 'mtime': 1257400876, 'rmajor': 0, 'machine': ('Linux', 'stkendca18a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 820618800L, 20L, 1, 1602, 2752, 512L, 1257400847, 1257400847, 1254398418), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 1966080000L, 'gname': 'root', 'gid': 0, 'fullname': '/diska/write-pool-1/data/0030000000000000061EB148', 'inode': 807317832L, 'minor': 20}, 'version': 'v3_8b  CVS $Revision: 1.922 $ <frozen>', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diska/write-pool-1/data/0030000000000000061EB148', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/.(access)(0030000000000000061EB148)', 'fc': {'size': 1966080000L, 'external_label': 'VON266', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/diska/write-pool-1/data/0030000000000000061EB148', 'at_the_top': 86, 'local_inode': 536955558L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.6', 54460), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257400918.22574, 'encp_start_time': 1257400874.730144, 'job_queued': 1257400876.62868, 't0': 1257400874, 'in_queue': 41.597016096115112}, 'user_level': 'user2', 'unique_id': 'stkendca18a.fnal.gov-1257400876-21257-0'} Thread client_connect_thread
00:01:58 stkendca18a.fnal.gov 021257 root I ENCP  Control socket ('131.225.13.6', 54460) is connected to ('131.225.13.201', 42204) for stkendca18a.fnal.gov-1257400876-21257-0.  elapsed=43.550sec
00:01:58 stkendca18a.fnal.gov 021257 root I ENCP  Received callback ticket from mover LTO4_105.mover for transfer stkendca18a.fnal.gov-1257400876-21257-0.  elapsed=43.560sec
00:01:58 stkendca18a.fnal.gov 021257 root I ENCP  Atempting to connect data socket to mover at ('131.225.13.201', 42203).
00:01:58 stkendca18a.fnal.gov 021257 root I ENCP  Connecting data socket to mover (('131.225.13.201', 42203)) with interface 131.225.13.6.  elapsed=43.560sec
00:01:58 stkendca18a.fnal.gov 021257 root I ENCP  Data socket ('131.225.13.6', 54465) is connected to ('131.225.13.201', 42203).  elapsed=43.560sec
00:01:58 stkendca18a.fnal.gov 021257 root I ENCP  Opening local file.
00:01:58 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:01:58 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  tape position: target 187 current 187 0 Thread seek_thread
00:01:58 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  write_tape starting, bytes_to_write=1966080512 Thread tape_thread
00:01:58 stkensrv0n.fnal.gov 028796 enstore I FILSRV  Updating  			update file set deleted = 'n', pnfs_path = '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702068213_s103263_p28', drive = 'stkenmvr205a:/dev/rmt/tps0d0n:9310801322', pnfs_id = '0030000000000000061EB0D8' where bfid = 'CDMS125740091700000';
00:01:58 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  Write starting. Tape VON266 absolute location in blocks 2634427 Thread tape_thread
00:01:58 stkendca18a.fnal.gov 021257 root I ENCP  pf_status = ('ok', None)  skip_layer_cleanup = False
00:01:58 stkendca18a.fnal.gov 020159 root I ENCP  MSG_TYPE=ENCP_XFER    write_to_hsm /diskc/write-pool-3/data/0030000000000000061EB0D8 -> /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702068213_s103263_p28: 1966080000 bytes copied to VON266 at 60.9 MB/S (76.4 MB/S network) (126 MB/S drive) (64.9 MB/S disk) mover=LTO4_105.mover drive_id=ULTRIUM-TD4 drive_sn=9310801322 drive_vendor=IBM elapsed=104.95 {'storage_group': 'exp-db', 'encp_crc': 4195779471L, 'disk_rate': 64.885846071559442, 'drive_rate': 126.06805830371974, 'driver': 'FTTDriver', 'transfer_rate': 60.949691634157418, 'mover_interface': 'stkenmvr205a', 'overall_rate': 60.114492350864417, 'media_changer': 'SL8500G1.media_changer', 'network_rate': 60.949691634157418, 'encp_ip': '131.225.13.6', 'unique_id': 'stkendca18a.fnal.gov-1257400814-20159-0'}
00:01:58 stkendca18a.fnal.gov 020159 root I ENCP  File /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702068213_s103263_p28 copied successfully.
00:01:58 stkendca18a.fnal.gov 020159 root I ENCP  Completed transferring 1966080000 bytes in 1 files in 104.950722933 sec.  Overall rate = 60.1 MB/sec.  Transfer rate = 60.9 MB/sec.  Network rate = 76.4 MB/sec.  Drive rate = 126 MB/sec.  Disk rate = 64.9 MB/sec.  Exit status = 0.
00:01:58 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  <storage groups {'ktev': [('9940B10.mover', 'VO6332'), ('9940B35.mover', 'VO6341')]} volume_families {'ktev.ktev_migration.cern': [('9940B10.mover', 'VO6332'), ('9940B35.mover', 'VO6341')]} >
00:01:58 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  read_from_hsm next work on vol=VO6341 mover=9940B35.mover requester:root
00:01:58 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  HAVE_BOUND:sending read_from_hsm /pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC083/f0 to mover 9940B35.mover ('131.225.13.87', 7835) DEL_DISM 2
00:01:58 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  MSG_TYPE=ADD_TO_LMQ  request added to work at movers queue for storage group : ktev
00:01:58 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  HAVE_BOUND:sending {'vc': {'comment': '', 'declared': 1083263105.0, 'blocksize': 131072, 'storage_group': 'ktev', 'library': 'CD-9940B', 'eod_cookie': '0000_000000000_0000172', 'current_location': '0000_000000000_0000023', 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'migrating'], 'external_label': 'VO6341', 'wrapper': 'cern', 'remaining_bytes': 6809693696L, 'sum_mounts': 54, 'sum_rd_access': 93, 'media_type': '9940B', 'last_access': 1257400550.0, 'status': ('ok', None), 'si_time': [0.0, 1257399311.0], 'capacity_bytes': 214748364800L, 'non_del_files': 57, 'sum_wr_err': 0, 'sum_wr_access': 57, 'file_family': 'ktev_migration', 'address': ('131.225.13.59', 7502), 'volume_family': 'ktev.ktev_migration.cern', 'modification_time': 1193243360.0, 'write_protected': 'y', 'sum_rd_err': 0, 'first_access': 1088544231.0}, 'outfilepath': '/data/data2/Migration_Spool/VO6341:0000_000000000_0000023', 'encp': {'agetime': 30, 'adminpri': 6, 'delpri': 1, 'basepri': 1000007, 'delayed_dismount': 2, 'curpri': 1000007}, 'file_size': 4242341652L, 'mover': '9940B35.mover', 'lm': {'address': ('131.225.13.4', 7522)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC083/f0', 'uid': 0, 'uname': 'root', 'gid': 0, 'rmajor': 0, 'machine': ('Linux', 'stkendm7a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (33188, 525142144L, 20L, 1, 5406, 2539, 1L, 1177383248, 1088548411, 1088546827), 'mode_octal': '0100744', 'mode': 33252, 'gname': 'root', 'size_bytes': 4242341652L, 'fullname': '/data/data2/Migration_Spool/VO6341:0000_000000000_0000023', 'inode': 525142144L, 'minor': 20}, 'version': 'v3_8  CVS $Revision: 1.963 $ encp.pyc', 'encp_daq': None, 'client_crc': 1, 'infile': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC083/.(access)(001F000000000000014D0880)', 'status': ('ok', None), 'volume': 'VO6341', 'outfile': '/data/data2/Migration_Spool/VO6341:0000_000000000_0000023', 'fc': {'status': ('ok', None), 'pnfsid': '001F000000000000014D0880', 'pnfs_name0': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC083/f0', 'deleted': 'no', 'external_label': 'VO6341', 'drive': 'stkenmvr10a:/dev/rmt/tps0d0n:4790000020', 'update': None, 'gid': 0, 'location_cookie': '0000_000000000_0000023', 'complete_crc': 701935742L, 'address': ('131.225.13.59', 7501), 'uid': 0, 'bfid': 'CDMS108854841100000', 'sanity_cookie': (65536L, 2360529580L), 'size': 4242341652L}, 'infilepath': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC083/f0', 'at_the_top': 6, 'bfid': 'CDMS108854841100000', 'local_inode': 4472841L, 'resend': {'max_retry': 3, 'resubmits': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.19', 50809), 'work': 'read_from_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257400918.5814681, 'encp_start_time': 1257400552.5200961, 'job_queued': 1257400552.7230361, 't0': 1257400552, 'in_queue': 365.85801196098328}, 'user_level': 'admin', 'unique_id': 'stkendm7a.fnal.gov-1257400552-29514-7'} to mover
00:01:58 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  <storage groups {'ktev': [('9940B10.mover', 'VO6332'), ('9940B35.mover', 'VO6341')]} volume_families {'ktev.ktev_migration.cern': [('9940B10.mover', 'VO6332'), ('9940B35.mover', 'VO6341')]} >
00:01:58 stkenmvr35a.fnal.gov 005964 root I DBT35MV  READ FROM HSM Thread MainThread
00:01:58 stkenmvr35a.fnal.gov 005964 root I DBT35MV  SENDING {'vc': {'comment': '', 'declared': 1083263105.0, 'blocksize': 131072, 'storage_group': 'ktev', 'library': 'CD-9940B', 'eod_cookie': '0000_000000000_0000172', 'current_location': '0000_000000000_0000023', 'user_inhibit': ['none', 'none'], 'sum_rd_access': 93, 'system_inhibit': ['none', 'migrating'], 'external_label': 'VO6341', 'wrapper': 'cern', 'remaining_bytes': 6809693696L, 'sum_mounts': 54, 'capacity_bytes': 214748364800L, 'media_type': '9940B', 'last_access': 1257400550.0, 'status': ('ok', None), 'si_time': [0.0, 1257399311.0], 'non_del_files': 57, 'sum_wr_err': 0, 'sum_wr_access': 57, 'file_family': 'ktev_migration', 'address': ('131.225.13.59', 7502), 'volume_family': 'ktev.ktev_migration.cern', 'modification_time': 1193243360.0, 'write_protected': 'y', 'sum_rd_err': 0, 'first_access': 1088544231.0}, 'outfilepath': '/data/data2/Migration_Spool/VO6341:0000_000000000_0000023', 'encp': {'agetime': 30, 'adminpri': 6, 'delpri': 1, 'basepri': 1000007, 'delayed_dismount': 2, 'curpri': 1000007}, 'file_size': 4242341652L, 'mover': {'max_buffer': 1048576000L, 'check_written_file': 10, 'max_failures': 5, 'library': 'CD-9940B.library_manager', 'local_mover': 0, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7835, 'mc_device': '0,0,10,1', 'compression': 0, 'serial_num': '479001001549', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 28311552.0, 'vendor_id': 'STK', 'driver': 'FTTDriver', 'host': 'stkenmvr35a', 'max_time_in_state': 1200, 'device': 'stkenmvr35a:/dev/rmt/tps2d0n', 'name': '9940B35.mover', 'send_stats': 1, 'product_id': 'T9940B', 'statistics_path': '/tmp/enstore/enstore/DBT35MV.stat', 'mover_address': ('131.225.13.87', 7835), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.87', 33888), 'data_ip': 'stkenmvr35a', 'logname': 'DBT35MV', 'hostip': '131.225.13.87', 'do_cleaning': 'No', 'media_changer': 'stk.media_changer'}, 'lm': {'address': ('131.225.13.4', 7522)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC083/f0', 'uid': 0, 'uname': 'root', 'rmajor': 0, 'machine': ('Linux', 'stkendm7a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (33188, 525142144L, 20L, 1, 5406, 2539, 1L, 1177383248, 1088548411, 1088546827), 'mode_octal': '0100744', 'mode': 33252, 'size_bytes': 4242341652L, 'gname': 'root', 'gid': 0, 'fullname': '/data/data2/Migration_Spool/VO6341:0000_000000000_0000023', 'inode': 525142144L, 'minor': 20}, 'version': 'v3_8  CVS $Revision: 1.963 $ encp.pyc', 'encp_daq': None, 'client_crc': 1, 'infile': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC083/.(access)(001F000000000000014D0880)', 'status': ('ok', None), 'volume': 'VO6341', 'outfile': '/data/data2/Migration_Spool/VO6341:0000_000000000_0000023', 'fc': {'status': ('ok', None), 'pnfsid': '001F000000000000014D0880', 'pnfs_name0': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC083/f0', 'deleted': 'no', 'external_label': 'VO6341', 'drive': 'stkenmvr10a:/dev/rmt/tps0d0n:4790000020', 'update': None, 'gid': 0, 'location_cookie': '0000_000000000_0000023', 'complete_crc': 701935742L, 'address': ('131.225.13.59', 7501), 'uid': 0, 'bfid': 'CDMS108854841100000', 'sanity_cookie': (65536L, 2360529580L), 'size': 4242341652L}, 'infilepath': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC083/f0', 'at_the_top': 6, 'bfid': 'CDMS108854841100000', 'local_inode': 4472841L, 'resend': {'max_retry': 3, 'resubmits': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.19', 50809), 'work': 'read_from_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257400918.5814681, 'encp_start_time': 1257400552.5200961, 'job_queued': 1257400552.7230361, 't0': 1257400552, 'in_queue': 365.85801196098328}, 'user_level': 'admin', 'unique_id': 'stkendm7a.fnal.gov-1257400552-29514-7'} Thread client_connect_thread
00:01:58 stkendm7a.fnal.gov 029514 root I ENCP  Listening for control socket at: ('131.225.13.19', 50809) Thread READ_1
00:01:58 stkendm7a.fnal.gov 029514 root I ENCP  Waiting for mover to connect control socket at address: ('131.225.13.19', 50809)  elapsed=1606.370sec Thread READ_1
00:01:58 stkendm7a.fnal.gov 029514 root I ENCP  Control socket ('131.225.13.19', 50809) is connected to ('131.225.13.87', 33889) for stkendm7a.fnal.gov-1257400552-29514-7.  elapsed=1606.380sec Thread READ_1
00:01:58 stkendm7a.fnal.gov 029514 root I ENCP  Received callback ticket from mover 9940B35.mover for transfer stkendm7a.fnal.gov-1257400552-29514-7.  elapsed=1606.380sec Thread READ_1
00:01:58 stkendm7a.fnal.gov 029514 root I ENCP  Attempting to connect data socket to mover at ('131.225.13.87', 33888). Thread READ_1
00:01:58 stkendm7a.fnal.gov 029514 root I ENCP  Connecting data socket to mover (('131.225.13.87', 33888)) with interface 131.225.13.19.  elapsed=1606.390sec Thread READ_1
00:01:58 stkendm7a.fnal.gov 029514 root I ENCP  Data socket ('131.225.13.19', 50828) is connected to ('131.225.13.87', 33888).  elapsed=1606.390sec Thread READ_1
00:01:58 stkendm7a.fnal.gov 029514 root I ENCP  Opening local file. Thread READ_1
00:01:58 stkenmvr35a.fnal.gov 005964 root I DBT35MV  tape position: target 23 current 20 1 Thread seek_thread
00:01:58 stkenmvr35a.fnal.gov 005964 root I DBT35MV  skip_fm 3 Thread seek_thread
00:01:58 stkenmvr35a.fnal.gov 005964 root I DBT35MV  read_tape starting, bytes_to_read=4242341652 Thread tape_thread
00:01:59 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:01:59 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  <storage groups {'ktev': [('9940B10.mover', 'VO6332'), ('9940B35.mover', 'VO6341')]} volume_families {'ktev.ktev_migration.cern': [('9940B10.mover', 'VO6332'), ('9940B35.mover', 'VO6341')]} >
00:01:59 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:02:00 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:02:01 last message repeated 1 times
00:02:01 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:02:02 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:02:03 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV  read bytes 2118453935/2118453935, blocks 16163 header 234 Thread tape_thread
00:02:03 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV  drive stats after read. Tape VOP775 position 1 block 16163 block_size 0 bloc_loc 16165 tot_blocks 762939000 BOT 0 read_err 0 write_err 0 bytes 2118453935 block_read_tot 17.3940792084 tape_rate 121791668.856 Thread tape_thread
00:02:03 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV  read_tape exiting, read 2118453935/2118453935 bytes Thread tape_thread
00:02:03 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV  read_tape timing:get_space 0.698738574982 read 8.16539788246 crc_check 8.37064146996 push 0.159301280975 block_read 17.3940792084 idle 0.310658216476 Thread tape_thread
00:02:03 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:02:03 stkensrv4n.fnal.gov 005357 enstore E LTO4G1LM  MOVER ERROR RQ {'status': ('ok', None), 'returned_work': None, 'library': None, 'mover_type': '', 'current_location': '0000_000000000_0000000', 'ip_map': '', 'operation': None, 'read_only': 0, 'current_time': 1257400923.86285, 'error_source': None, 'external_label': None, 'state': 'OFFLINE', 'r_a': (('131.225.13.194', 54880), 16280L, '131.225.13.194-54880-1256586929.509013-4866'), 'library_list': [('CD-LTO4G1.library_manager', ('131.225.13.4', 8541))], 'current_priority': (None, None), 'transfer_deficiency': 1, 'time_in_state': 813993.50279188156, 'address': ('131.225.13.194', 7860), 'volume_family': None, 'work': 'mover_error', 'mover': 'LTO4_99.mover', 'volume_status': (['none', 'none'], ['none', 'none']), 'unique_id': None}
00:02:04 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:02:05 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:02:06 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:02:07 stkendca18a.fnal.gov 022164 root I ENCP  Version: v3_8b  CVS $Revision: 1.922 $ <frozen>  OS: Linux 2.6.9-89.0.9.ELsmp i686 Release:  Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0)  Group: root(0)  Euser: root(0)  Egroup: root(0)  Library: CD-LTO4G1  Storage Group: exp-db  File Family: daily-d0-offline  FF Wrapper: cpio_odc  FF Width: 2  Current working directory: stkendca18a.fnal.gov:/home/enstore/dcache-code/dcache-fermi-config  Command line: /fnal/ups/prd/encp/v3_8b/Linux-2-6-dcache/encp --verbose 4 --threaded --ecrc --bypass-filesystem-max-filesize-check --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702068213_s103263_p3 --put-cache 0030000000000000061EB018 /diskb/write-pool-2/data/0030000000000000061EB018
00:02:07 stkendca18a.fnal.gov 022164 root I ENCP  Creating zero length output files.
00:02:07 stkendca18a.fnal.gov 022163 root I ENCP  Version: v3_8b  CVS $Revision: 1.922 $ <frozen>  OS: Linux 2.6.9-89.0.9.ELsmp i686 Release:  Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0)  Group: root(0)  Euser: root(0)  Egroup: root(0)  Library: CD-LTO4G1  Storage Group: exp-db  File Family: daily-d0-offline  FF Wrapper: cpio_odc  FF Width: 2  Current working directory: stkendca18a.fnal.gov:/home/enstore/dcache-code/dcache-fermi-config  Command line: /fnal/ups/prd/encp/v3_8b/Linux-2-6-dcache/encp --verbose 4 --threaded --ecrc --bypass-filesystem-max-filesize-check --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702065710_s103261_p31 --put-cache 0030000000000000061EB048 /diskb/write-pool-2/data/0030000000000000061EB048
00:02:07 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:02:07 stkendca18a.fnal.gov 022164 root I ENCP  Sending /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702068213_s103263_p3 write request to LM: unique_id: stkendca18a.fnal.gov-1257400927-22164-0 inputfile: /diskb/write-pool-2/data/0030000000000000061EB018 outputfile: /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/.(access)(0030000000000000061EB018)
00:02:07 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  MSG_TYPE=ADD_TO_LMQ  request added to pending queue for storage group : exp-db
00:02:07 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  write Q'd /diskb/write-pool-2/data/0030000000000000061EB018 (stkendca18a.fnal.gov-1257400927-22164-0) -> /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702068213_s103263_p3 : library=CD-LTO4G1 family=daily-d0-offline requester:root volume_family:exp-db.daily-d0-offline.cpio_odc
00:02:07 stkendca18a.fnal.gov 022164 root I ENCP  Listening for control socket at: ('131.225.13.6', 54468)
00:02:07 stkendca18a.fnal.gov 022164 root I ENCP  Waiting for mover to connect control socket at address: ('131.225.13.6', 54468)  elapsed=1.580sec
00:02:07 stkendca18a.fnal.gov 022163 root I ENCP  Creating zero length output files.
00:02:07 stkendca18a.fnal.gov 022163 root I ENCP  Sending /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702065710_s103261_p31 write request to LM: unique_id: stkendca18a.fnal.gov-1257400927-22163-0 inputfile: /diskb/write-pool-2/data/0030000000000000061EB048 outputfile: /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/.(access)(0030000000000000061EB048)
00:02:07 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  MSG_TYPE=ADD_TO_LMQ  request added to pending queue for storage group : exp-db
00:02:07 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  write Q'd /diskb/write-pool-2/data/0030000000000000061EB048 (stkendca18a.fnal.gov-1257400927-22163-0) -> /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702065710_s103261_p31 : library=CD-LTO4G1 family=daily-d0-offline requester:root volume_family:exp-db.daily-d0-offline.cpio_odc
00:02:07 stkendca18a.fnal.gov 022163 root I ENCP  Listening for control socket at: ('131.225.13.6', 54469)
00:02:07 stkendca18a.fnal.gov 022163 root I ENCP  Waiting for mover to connect control socket at address: ('131.225.13.6', 54469)  elapsed=1.610sec
00:02:08 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:02:08 stkensrv3n.fnal.gov 020124 root I ENCP  File /pnfs/cms/WAX/7/PHEDEX2/PCP04/Digi/eg03_jets_2g_pt50170.1/ZippedEVD.63703329.63703340.eg_2x1033PU761_TkMu_2_g133_OSC.eg03_jets_2g_pt50170.zip transfered for stkensrv3n.fnal.gov-1257400800-20124-0 in 24.1243438721 seconds.
00:02:08 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV  write_client: wrote 2118453701/2118453701 bytes Thread net_thread
00:02:08 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV  write_client exits Thread net_thread
00:02:08 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV  transfer complete volume=VOP775 location=1 Thread net_thread
00:02:08 last message repeated 1 times
00:02:08 stkensrv0n.fnal.gov 028762 enstore I VOLSRV  Updating  			update volume set sum_rd_access = '1447', last_access = '2009-11-05 00:02:08' where label = 'VOP775';
00:02:08 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV  Sending done to client: {'vc': {'comment': '<= VO5281 VO7669 VO9276 VO9363 VO9434 VO9484', 'declared': 1231186074.0, 'blocksize': 131072, 'storage_group': 'cms', 'library': 'CD-LTO4G1', 'eod_cookie': '0000_000000000_0001446', 'current_location': '0000_000000000_0000001', 'user_inhibit': ['none', 'none'], 'sum_rd_access': 1446, 'system_inhibit': ['none', 'full'], 'external_label': 'VOP775', 'wrapper': 'cpio_odc', 'remaining_bytes': 1525791744L, 'sum_mounts': 6, 'capacity_bytes': 858993459200L, 'media_type': 'LTO4', 'last_access': 1257400798.0, 'status': ('ok', None), 'si_time': [0.0, 1235726759.0], 'non_del_files': 1481, 'sum_wr_err': 0, 'sum_wr_access': 1445, 'file_family': 'cms7', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.cms7.cpio_odc', 'modification_time': 1238376412.0, 'write_protected': 'y', 'sum_rd_err': 0, 'first_access': 1235693914.0}, 'outfilepath': '/dev/null', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': 1, 'curpri': 1}, 'file_size': 2118453701L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'dismount_delay': 30, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7846, 'mc_device': '0,6,1,12', 'compression': 0, 'serial_num': '9311032672', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr146a', 'max_time_in_state': 1200, 'device': 'stkenmvr146a:/dev/rmt/tps0d0n', 'name': 'LTO4_46.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_46.stat', 'single_filemark': 1, 'mount_delay': 15, 'mover_address': ('131.225.13.174', 7846), 'check_first_written_file': 0, 'callback_addr': ('131.225.13.174', 44192), 'data_ip': 'stkenmvr146a', 'logname': 'LTO4_46MV', 'hostip': '131.225.13.174', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer', 'min_buffer': 67108864L}, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/cms/WAX/7/PHEDEX2/PCP04/Digi/eg03_jets_2g_pt50170.1/ZippedEVD.63703329.63703340.eg_2x1033PU761_TkMu_2_g133_OSC.eg03_jets_2g_pt50170.zip', 'uid': 0, 'uname': 'root', 'rmajor': 0, 'machine': ('Linux', 'stkensrv3n.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 08:50:41 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'pstat': (33188, 47319392L, 20L, 1, 9811, 5063, 2118453701L, 1238357363, 1132850253, 1132849993), 'mode_octal': '0100744', 'mode': 33252, 'size_bytes': 2118453701L, 'gname': 'root', 'gid': 0, 'fullname': '/dev/null', 'inode': 47319392L, 'minor': 20}, 'version': 'v3_7d  CVS $Revision: 1.929 $ encp', 'encp_daq': None, 'client_crc': 1, 'infile': '/pnfs/cms/WAX/7/PHEDEX2/PCP04/Digi/eg03_jets_2g_pt50170.1/.(access)(000200000000000000D20960)', 'status': ('ok', None), 'volume': 'VOP775', 'outfile': '/dev/null', 'fc': {'status': ('ok', None), 'pnfsid': '000200000000000000D20960', 'pnfs_name0': '/pnfs/fs/usr/cms/WAX/7/PHEDEX2/PCP04/Digi/eg03_jets_2g_pt50170.1/ZippedEVD.63703329.63703340.eg_2x1033PU761_TkMu_2_g133_OSC.eg03_jets_2g_pt50170.zip', 'deleted': 'no', 'external_label': 'VOP775', 'drive': 'stkenmvr147a:/dev/rmt/tps0d0n:1310051195', 'update': '2009-02-26 18:19:54.189315', 'gid': 0, 'location_cookie': '0000_000000000_0000001', 'complete_crc': 4130209940L, 'address': ('131.225.13.59', 7501), 'uid': 0, 'bfid': 'CDMS123569399100000', 'sanity_cookie': (65536L, 1817439004L), 'size': 2118453701L}, 'infilepath': '/pnfs/cms/WAX/7/PHEDEX2/PCP04/Digi/eg03_jets_2g_pt50170.1/ZippedEVD.63703329.63703340.eg_2x1033PU761_TkMu_2_g133_OSC.eg03_jets_2g_pt50170.zip', 'at_the_top': 34, 'bfid': 'CDMS123569399100000', 'local_inode': 2469L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': 7, 'retry': 0}, 'callback_addr': ('131.225.13.51', 36441), 'work': 'read_from_hsm', 'override_ro_mount': 1, 'times': {'encp_start_time': 1257400799.204268, 'seek_time': 102.49328017234802, 'transfer_time': 24.30(trunc. 4409) Thread net_thread
00:02:08 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV  Done is sent Thread net_thread
00:02:08 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV  maybe_clean Thread net_thread
00:02:08 stkensrv3n.fnal.gov 020124 root I ENCP  Received final dialog for stkensrv3n.fnal.gov-1257400800-20124-0.  elapsed=129.780sec
00:02:08 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV  returned from maybe_clean Thread net_thread
00:02:09 stkensrv3n.fnal.gov 020124 root I ENCP  leaving transfer_file().
00:02:09 stkensrv3n.fnal.gov 020124 root I ENCP  MSG_TYPE=ENCP_XFER    read_from_hsm /pnfs/cms/WAX/7/PHEDEX2/PCP04/Digi/eg03_jets_2g_pt50170.1/ZippedEVD.63703329.63703340.eg_2x1033PU761_TkMu_2_g133_OSC.eg03_jets_2g_pt50170.zip -> /dev/null: 2118453701 bytes copied from VOP775 at 83.3 MB/S (83.7 MB/S network) (115 MB/S drive) (83.7 MB/S disk) mover=LTO4_46.mover drive_id=ULTRIUM-TD4 drive_sn=9311032672 drive_vendor=IBM elapsed=129.79 {'storage_group': 'cms', 'encp_crc': 4130209940L, 'disk_rate': 83.746382078629679, 'drive_rate': 114.95892969242279, 'driver': 'FTTDriver', 'transfer_rate': 83.287932725232949, 'mover_interface': 'stkenmvr146a', 'overall_rate': 15.917678691428964, 'media_changer': 'SL8500G1.media_changer', 'network_rate': 83.287932725232949, 'encp_ip': '131.225.13.51', 'unique_id': 'stkensrv3n.fnal.gov-1257400800-20124-0'}
00:02:09 stkensrv3n.fnal.gov 020124 root I ENCP  File /dev/null copied successfully.
00:02:09 stkensrv3n.fnal.gov 020124 root I ENCP  Completed transferring 2118453701 bytes in 1 files in 129.801612139 sec.  Overall rate = 15.9 MB/sec.  Transfer rate = 83.3 MB/sec.  Network rate = 83.7 MB/sec.  Drive rate = 115 MB/sec.  Disk rate = 83.7 MB/sec.  Exit status = 0.
00:02:09 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:02:09 stkensrv3n.fnal.gov 020327 root I ENCP  Version: v3_7d  CVS $Revision: 1.929 $ encp  OS: Linux 2.6.9-89.0.9.ELsmp x86_64 Release:  Scientific Linux SL release 4.4 (Beryllium), User: root(0)  Group: root(0)  Euser: root(0)  Egroup: root(0)  Current working directory: stkensrv3n.fnal.gov:/root  Command line: /opt/enstore/bin/encp --verbose 10 --bypass-filesystem-max-filesize-check --max-resubmit 7 --delayed-dismount 1 --override-ro-mount --get-bfid CDMS123572675300001 /dev/null
00:02:09 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:02:10 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:02:10 stkensrv3n.fnal.gov 020327 root I ENCP  Creating zero length output files.
00:02:10 stkensrv3n.fnal.gov 020327 root I ENCP  Sending /pnfs/cms/WAX/7/PCP04/Hit/eg03_jets_2g_pt50170/EVD0_Events.56914b76158411d998e90002b3d8e854.63719000.eg03_jets_2g_pt50170.eg_Hit245_2_g133 read request to LM: unique_id: stkensrv3n.fnal.gov-1257400930-20327-0 inputfile: /pnfs/cms/WAX/7/PCP04/Hit/eg03_jets_2g_pt50170/.(access)(00020000000000000008F760) outputfile: /dev/null
00:02:10 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  MSG_TYPE=ADD_TO_LMQ  request added to pending queue for storage group : cms
00:02:10 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  read Q'd /pnfs/cms/WAX/7/PCP04/Hit/eg03_jets_2g_pt50170/EVD0_Events.56914b76158411d998e90002b3d8e854.63719000.eg03_jets_2g_pt50170.eg_Hit245_2_g133 (stkensrv3n.fnal.gov-1257400930-20327-0) -> /dev/null : library=CD-LTO4G1 family=cms.cms7.cpio_odc requester:root
00:02:11 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:02:11 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  read_from_hsm next work on vol=VOP775 mover=LTO4_46.mover requester:root
00:02:11 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  HAVE_BOUND:sending read_from_hsm /pnfs/cms/WAX/7/PCP04/Hit/eg03_jets_2g_pt50170/EVD0_Events.56914b76158411d998e90002b3d8e854.63719000.eg03_jets_2g_pt50170.eg_Hit245_2_g133 to mover LTO4_46.mover ('131.225.13.174', 7846) DEL_DISM 1
00:02:11 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  MSG_TYPE=ADD_TO_LMQ  request added to work at movers queue for storage group : cms
00:02:11 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  HAVE_BOUND:sending {'vc': {'comment': '<= VO5281 VO7669 VO9276 VO9363 VO9434 VO9484', 'declared': 1231186074.0, 'blocksize': 131072, 'storage_group': 'cms', 'library': 'CD-LTO4G1', 'eod_cookie': '0000_000000000_0001446', 'current_location': '0000_000000000_0001445', 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'full'], 'external_label': 'VOP775', 'wrapper': 'cpio_odc', 'remaining_bytes': 1525791744L, 'sum_mounts': 6, 'sum_rd_access': 1447, 'media_type': 'LTO4', 'last_access': 1257400928.0, 'status': ('ok', None), 'si_time': [0.0, 1235726759.0], 'capacity_bytes': 858993459200L, 'non_del_files': 1481, 'sum_wr_err': 0, 'sum_wr_access': 1445, 'file_family': 'cms7', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.cms7.cpio_odc', 'modification_time': 1238376412.0, 'write_protected': 'y', 'sum_rd_err': 0, 'first_access': 1235693914.0}, 'outfilepath': '/dev/null', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': 1, 'curpri': 1}, 'file_size': 547296L, 'mover': 'LTO4_46.mover', 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/cms/WAX/7/PCP04/Hit/eg03_jets_2g_pt50170/EVD0_Events.56914b76158411d998e90002b3d8e854.63719000.eg03_jets_2g_pt50170.eg_Hit245_2_g133', 'uid': 0, 'uname': 'root', 'gid': 0, 'rmajor': 0, 'machine': ('Linux', 'stkensrv3n.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 08:50:41 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'pstat': (33188, 34142048L, 20L, 1, 9811, 5063, 547296L, 1238376412, 1113792704, 1113792251), 'mode_octal': '0100744', 'mode': 33252, 'gname': 'root', 'size_bytes': 547296L, 'fullname': '/dev/null', 'inode': 34142048L, 'minor': 20}, 'version': 'v3_7d  CVS $Revision: 1.929 $ encp', 'encp_daq': None, 'client_crc': 1, 'infile': '/pnfs/cms/WAX/7/PCP04/Hit/eg03_jets_2g_pt50170/.(access)(00020000000000000008F760)', 'status': ('ok', None), 'volume': 'VOP775', 'outfile': '/dev/null', 'fc': {'status': ('ok', None), 'pnfsid': '00020000000000000008F760', 'pnfs_name0': '/pnfs/fs/usr/cms/WAX/7/PCP04/Hit/eg03_jets_2g_pt50170/EVD0_Events.56914b76158411d998e90002b3d8e854.63719000.eg03_jets_2g_pt50170.eg_Hit245_2_g133', 'deleted': 'no', 'external_label': 'VOP775', 'drive': 'stkenmvr148a:/dev/rmt/tps0d0n:9310060094', 'update': '2009-02-27 03:25:56.475019', 'gid': 0, 'location_cookie': '0000_000000000_0001445', 'complete_crc': 1644974717L, 'address': ('131.225.13.59', 7501), 'uid': 0, 'bfid': 'CDMS123572675300001', 'sanity_cookie': (65536L, 3819061325L), 'size': 547296L}, 'infilepath': '/pnfs/cms/WAX/7/PCP04/Hit/eg03_jets_2g_pt50170/EVD0_Events.56914b76158411d998e90002b3d8e854.63719000.eg03_jets_2g_pt50170.eg_Hit245_2_g133', 'at_the_top': 9, 'bfid': 'CDMS123572675300001', 'local_inode': 2469L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': 7, 'retry': 0}, 'callback_addr': ('131.225.13.51', 36450), 'work': 'read_from_hsm', 'override_ro_mount': 1, 'times': {'lm_dequeued': 1257400931.164377, 'encp_start_time': 1257400929.1784661, 'job_queued': 1257400930.4335001, 't0': 1257400929, 'in_queue': 0.73083686828613281}, 'user_level': 'admin', 'unique_id': 'stkensrv3n.fnal.gov-1257400930-20327-0'} to mover
00:02:11 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:02:11 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV  READ FROM HSM Thread MainThread
00:02:11 stkensrv3n.fnal.gov 020327 root I ENCP  Listening for control socket at: ('131.225.13.51', 36450)
00:02:11 stkensrv3n.fnal.gov 020327 root I ENCP  Waiting for mover to connect control socket at address: ('131.225.13.51', 36450)  elapsed=2.010sec
00:02:11 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:02:11 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV  SENDING {'vc': {'comment': '<= VO5281 VO7669 VO9276 VO9363 VO9434 VO9484', 'declared': 1231186074.0, 'blocksize': 131072, 'storage_group': 'cms', 'library': 'CD-LTO4G1', 'eod_cookie': '0000_000000000_0001446', 'current_location': '0000_000000000_0001445', 'user_inhibit': ['none', 'none'], 'sum_rd_access': 1447, 'system_inhibit': ['none', 'full'], 'external_label': 'VOP775', 'wrapper': 'cpio_odc', 'remaining_bytes': 1525791744L, 'sum_mounts': 6, 'capacity_bytes': 858993459200L, 'media_type': 'LTO4', 'last_access': 1257400928.0, 'status': ('ok', None), 'si_time': [0.0, 1235726759.0], 'non_del_files': 1481, 'sum_wr_err': 0, 'sum_wr_access': 1445, 'file_family': 'cms7', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.cms7.cpio_odc', 'modification_time': 1238376412.0, 'write_protected': 'y', 'sum_rd_err': 0, 'first_access': 1235693914.0}, 'outfilepath': '/dev/null', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': 1, 'curpri': 1}, 'file_size': 547296L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'dismount_delay': 30, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7846, 'mc_device': '0,6,1,12', 'compression': 0, 'serial_num': '9311032672', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr146a', 'max_time_in_state': 1200, 'device': 'stkenmvr146a:/dev/rmt/tps0d0n', 'name': 'LTO4_46.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_46.stat', 'single_filemark': 1, 'mount_delay': 15, 'mover_address': ('131.225.13.174', 7846), 'check_first_written_file': 0, 'callback_addr': ('131.225.13.174', 44200), 'data_ip': 'stkenmvr146a', 'logname': 'LTO4_46MV', 'hostip': '131.225.13.174', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer', 'min_buffer': 67108864L}, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/cms/WAX/7/PCP04/Hit/eg03_jets_2g_pt50170/EVD0_Events.56914b76158411d998e90002b3d8e854.63719000.eg03_jets_2g_pt50170.eg_Hit245_2_g133', 'uid': 0, 'uname': 'root', 'rmajor': 0, 'machine': ('Linux', 'stkensrv3n.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 08:50:41 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'pstat': (33188, 34142048L, 20L, 1, 9811, 5063, 547296L, 1238376412, 1113792704, 1113792251), 'mode_octal': '0100744', 'mode': 33252, 'size_bytes': 547296L, 'gname': 'root', 'gid': 0, 'fullname': '/dev/null', 'inode': 34142048L, 'minor': 20}, 'version': 'v3_7d  CVS $Revision: 1.929 $ encp', 'encp_daq': None, 'client_crc': 1, 'infile': '/pnfs/cms/WAX/7/PCP04/Hit/eg03_jets_2g_pt50170/.(access)(00020000000000000008F760)', 'status': ('ok', None), 'volume': 'VOP775', 'outfile': '/dev/null', 'fc': {'status': ('ok', None), 'pnfsid': '00020000000000000008F760', 'pnfs_name0': '/pnfs/fs/usr/cms/WAX/7/PCP04/Hit/eg03_jets_2g_pt50170/EVD0_Events.56914b76158411d998e90002b3d8e854.63719000.eg03_jets_2g_pt50170.eg_Hit245_2_g133', 'deleted': 'no', 'external_label': 'VOP775', 'drive': 'stkenmvr148a:/dev/rmt/tps0d0n:9310060094', 'update': '2009-02-27 03:25:56.475019', 'gid': 0, 'location_cookie': '0000_000000000_0001445', 'complete_crc': 1644974717L, 'address': ('131.225.13.59', 7501), 'uid': 0, 'bfid': 'CDMS123572675300001', 'sanity_cookie': (65536L, 3819061325L), 'size': 547296L}, 'infilepath': '/pnfs/cms/WAX/7/PCP04/Hit/eg03_jets_2g_pt50170/EVD0_Events.56914b76158411d998e90002b3d8e854.63719000.eg03_jets_2g_pt50170.eg_Hit245_2_g133', 'at_the_top': 9, 'bfid': 'CDMS123572675300001', 'local_inode': 2469L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': 7, 'retry': 0}, 'callback_addr': ('131.225.13.51', 36450), 'work': 'read_from_hsm', 'override_ro_mount': 1, 'times': {'lm_dequeued': 1257400931.164377, 'encp_start_time': 1257400929.1784661, 'job_queued': 1257400930.4335001, 't0': 1257400929, 'in_queue': 0.7308368(trunc. 4170) Thread client_connect_thread
00:02:11 stkensrv3n.fnal.gov 020327 root I ENCP  Control socket ('131.225.13.51', 36450) is connected to ('131.225.13.174', 44201) for stkensrv3n.fnal.gov-1257400930-20327-0.  elapsed=2.040sec
00:02:11 stkensrv3n.fnal.gov 020327 root I ENCP  Received callback ticket from mover LTO4_46.mover for transfer stkensrv3n.fnal.gov-1257400930-20327-0.  elapsed=2.040sec
00:02:11 stkensrv3n.fnal.gov 020327 root I ENCP  Attempting to connect data socket to mover at ('131.225.13.174', 44200).
00:02:11 stkensrv3n.fnal.gov 020327 root I ENCP  Connecting data socket to mover (('131.225.13.174', 44200)) with interface 131.225.13.51.  elapsed=2.040sec
00:02:11 stkensrv3n.fnal.gov 020327 root I ENCP  Data socket ('131.225.13.51', 36451) is connected to ('131.225.13.174', 44200).  elapsed=2.040sec
00:02:11 stkensrv3n.fnal.gov 020327 root I ENCP  Opening local file.
00:02:11 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV  tape position: target 1445 current 1 16163 Thread seek_thread
00:02:11 stkendca20a.fnal.gov 010464 root I ENCP  Version: v3_8b  CVS $Revision: 1.922 $ <frozen>  OS: Linux 2.6.9-89.0.9.ELsmp i686 Release:  Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0)  Group: root(0)  Euser: root(0)  Egroup: root(0)  Library: CD-LTO4G1  Storage Group: exp-db  File Family: daily-d0-offline  FF Wrapper: cpio_odc  FF Width: 2  Current working directory: stkendca20a.fnal.gov:/home/enstore/dcache-code/dcache-fermi-config  Command line: /fnal/ups/prd/encp/v3_8b/Linux-2-6-dcache/encp --verbose 4 --threaded --ecrc --bypass-filesystem-max-filesize-check --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702065710_s103260_p16 --put-cache 0030000000000000061EB1F0 /diska/write-pool-1/data/0030000000000000061EB1F0
00:02:11 stkendca20a.fnal.gov 010464 root I ENCP  Creating zero length output files.
00:02:11 stkendca20a.fnal.gov 010464 root I ENCP  Sending /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702065710_s103260_p16 write request to LM: unique_id: stkendca20a.fnal.gov-1257400931-10464-0 inputfile: /diska/write-pool-1/data/0030000000000000061EB1F0 outputfile: /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/.(access)(0030000000000000061EB1F0)
00:02:11 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  MSG_TYPE=ADD_TO_LMQ  request added to pending queue for storage group : exp-db
00:02:11 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  write Q'd /diska/write-pool-1/data/0030000000000000061EB1F0 (stkendca20a.fnal.gov-1257400931-10464-0) -> /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702065710_s103260_p16 : library=CD-LTO4G1 family=daily-d0-offline requester:root volume_family:exp-db.daily-d0-offline.cpio_odc
00:02:11 stkendca20a.fnal.gov 010464 root I ENCP  Listening for control socket at: ('131.225.13.12', 40605)
00:02:11 stkendca20a.fnal.gov 010464 root I ENCP  Waiting for mover to connect control socket at address: ('131.225.13.12', 40605)  elapsed=0.960sec
00:02:11 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:02:12 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:02:12 stkendca17a.fnal.gov 031426 root I ENCP  Version: v3_8b  CVS $Revision: 1.922 $ <frozen>  OS: Linux 2.6.9-89.0.9.ELsmp i686 Release:  Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0)  Group: root(0)  Euser: root(0)  Egroup: root(0)  Library: CD-LTO4G1  Storage Group: exp-db  File Family: daily-d0-offline  FF Wrapper: cpio_odc  FF Width: 2  Current working directory: stkendca17a.fnal.gov:/opt/d-cache/config  Command line: /fnal/ups/prd/encp/v3_8b/Linux-2-6-dcache/encp --verbose 4 --threaded --ecrc --bypass-filesystem-max-filesize-check --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702065710_s103261_p36 --put-cache 0030000000000000061EB200 /diska/write-pool-1/data/0030000000000000061EB200
00:02:12 stkendca17a.fnal.gov 031426 root I ENCP  Creating zero length output files.
00:02:12 stkendca17a.fnal.gov 031426 root I ENCP  Sending /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702065710_s103261_p36 write request to LM: unique_id: stkendca17a.fnal.gov-1257400932-31426-0 inputfile: /diska/write-pool-1/data/0030000000000000061EB200 outputfile: /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/.(access)(0030000000000000061EB200)
00:02:12 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  MSG_TYPE=ADD_TO_LMQ  request added to pending queue for storage group : exp-db
00:02:12 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  write Q'd /diska/write-pool-1/data/0030000000000000061EB200 (stkendca17a.fnal.gov-1257400932-31426-0) -> /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702065710_s103261_p36 : library=CD-LTO4G1 family=daily-d0-offline requester:root volume_family:exp-db.daily-d0-offline.cpio_odc
00:02:12 stkendca17a.fnal.gov 031426 root I ENCP  Listening for control socket at: ('131.225.13.5', 44354)
00:02:12 stkendca17a.fnal.gov 031426 root I ENCP  Waiting for mover to connect control socket at address: ('131.225.13.5', 44354)  elapsed=0.960sec
00:02:13 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:02:14 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:02:15 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:02:16 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:02:17 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:02:18 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:02:19 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:02:20 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:02:21 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:02:22 stkendca22a.fnal.gov 018233 root I ENCP  Version: v3_8b  CVS $Revision: 1.922 $ <frozen>  OS: Linux 2.6.9-89.0.9.ELsmp i686 Release:  Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0)  Group: root(0)  Euser: root(0)  Egroup: root(0)  Library: garbage  Storage Group: test  File Family: dcache  FF Wrapper: cpio_odc  FF Width: 1  Current working directory: stkendca22a.fnal.gov:/home/enstore/dcache-log  Command line: /fnal/ups/prd/encp/v3_8b/Linux-2-6-dcache/encp --verbose 4 --threaded --ecrc --bypass-filesystem-max-filesize-check --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0001 --put-cache 0005000000000000005FA590 /diskb/write-pool-2/data/0005000000000000005FA590
00:02:22 stkendca22a.fnal.gov 018289 root I ENCP  Version: v3_8b  CVS $Revision: 1.922 $ <frozen>  OS: Linux 2.6.9-89.0.9.ELsmp i686 Release:  Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0)  Group: root(0)  Euser: root(0)  Egroup: root(0)  Library: CD-LTO4G1  Storage Group: minos  File Family: minos  FF Wrapper: cpio_odc  FF Width: 1  Current working directory: stkendca22a.fnal.gov:/home/enstore/dcache-log  Command line: /fnal/ups/prd/encp/v3_8b/Linux-2-6-dcache/encp --verbose 4 --threaded --ecrc --bypass-filesystem-max-filesize-check --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/711/n13047113_0003_L010185R_D07_r4.cand.dogwood1.0.root --put-cache 000F0000000000000B2472B8 /diskb/write-pool-2/data/000F0000000000000B2472B8
00:02:22 stkendca22a.fnal.gov 018319 root I ENCP  Version: v3_8b  CVS $Revision: 1.922 $ <frozen>  OS: Linux 2.6.9-89.0.9.ELsmp i686 Release:  Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0)  Group: root(0)  Euser: root(0)  Egroup: root(0)  Library: garbage  Storage Group: test  File Family: dcache  FF Wrapper: cpio_odc  FF Width: 1  Current working directory: stkendca22a.fnal.gov:/home/enstore/dcache-log  Command line: /fnal/ups/prd/encp/v3_8b/Linux-2-6-dcache/encp --verbose 4 --threaded --ecrc --bypass-filesystem-max-filesize-check --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0003 --put-cache 0005000000000000005FA5F8 /diskb/write-pool-2/data/0005000000000000005FA5F8
00:02:22 stkendca22a.fnal.gov 018348 root I ENCP  Version: v3_8b  CVS $Revision: 1.922 $ <frozen>  OS: Linux 2.6.9-89.0.9.ELsmp i686 Release:  Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0)  Group: root(0)  Euser: root(0)  Egroup: root(0)  Library: garbage  Storage Group: test  File Family: dcache  FF Wrapper: cpio_odc  FF Width: 1  Current working directory: stkendca22a.fnal.gov:/home/enstore/dcache-log  Command line: /fnal/ups/prd/encp/v3_8b/Linux-2-6-dcache/encp --verbose 4 --threaded --ecrc --bypass-filesystem-max-filesize-check --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0002 --put-cache 0005000000000000005FA5C8 /diskb/write-pool-2/data/0005000000000000005FA5C8
00:02:22 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:02:22 stkendca22a.fnal.gov 018289 root I ENCP  Creating zero length output files.
00:02:22 stkendca22a.fnal.gov 018289 root I ENCP  Sending /pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/711/n13047113_0003_L010185R_D07_r4.cand.dogwood1.0.root write request to LM: unique_id: stkendca22a.fnal.gov-1257400942-18289-0 inputfile: /diskb/write-pool-2/data/000F0000000000000B2472B8 outputfile: /pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/711/.(access)(000F0000000000000B2472B8)
00:02:22 stkendca22a.fnal.gov 018233 root I ENCP  Configuration Server: no such name: 'garbage.library_manager'
00:02:22 stkendca22a.fnal.gov 018233 root E ENCP  INFILE=/diskb/write-pool-2/data/0005000000000000005FA590 OUTFILE=/pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0001 FILESIZE=801512 LABEL= LOCATION= DRIVE= DRIVE_SN= TRANSFER_TIME=0.00 SEEK_TIME=0.00 MOUNT_TIME=0.00 QWAIT_TIME=0.00 TIME2NOW=0.94 CRC= STATUS=KEYERROR Configuration Server: no such name: 'garbage.library_manager'
00:02:22 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  MSG_TYPE=ADD_TO_LMQ  request added to pending queue for storage group : minos
00:02:22 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  write Q'd /diskb/write-pool-2/data/000F0000000000000B2472B8 (stkendca22a.fnal.gov-1257400942-18289-0) -> /pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/711/n13047113_0003_L010185R_D07_r4.cand.dogwood1.0.root : library=CD-LTO4G1 family=minos requester:root volume_family:minos.minos.cpio_odc
00:02:22 stkendca22a.fnal.gov 018289 root I ENCP  Listening for control socket at: ('131.225.13.100', 40215)
00:02:22 stkendca22a.fnal.gov 018289 root I ENCP  Waiting for mover to connect control socket at address: ('131.225.13.100', 40215)  elapsed=0.950sec
00:02:22 stkendca22a.fnal.gov 018319 root I ENCP  Configuration Server: no such name: 'garbage.library_manager'
00:02:22 stkendca22a.fnal.gov 018319 root E ENCP  INFILE=/diskb/write-pool-2/data/0005000000000000005FA5F8 OUTFILE=/pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0003 FILESIZE=801512 LABEL= LOCATION= DRIVE= DRIVE_SN= TRANSFER_TIME=0.00 SEEK_TIME=0.00 MOUNT_TIME=0.00 QWAIT_TIME=0.00 TIME2NOW=0.97 CRC= STATUS=KEYERROR Configuration Server: no such name: 'garbage.library_manager'
00:02:22 stkendca22a.fnal.gov 018348 root I ENCP  Configuration Server: no such name: 'garbage.library_manager'
00:02:22 stkendca22a.fnal.gov 018348 root E ENCP  INFILE=/diskb/write-pool-2/data/0005000000000000005FA5C8 OUTFILE=/pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0002 FILESIZE=801512 LABEL= LOCATION= DRIVE= DRIVE_SN= TRANSFER_TIME=0.00 SEEK_TIME=0.00 MOUNT_TIME=0.00 QWAIT_TIME=0.00 TIME2NOW=0.93 CRC= STATUS=KEYERROR Configuration Server: no such name: 'garbage.library_manager'
00:02:23 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:02:24 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:02:24 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  write_to_hsm next work on vol=VON262 mover=LTO4_109.mover requester:root
00:02:24 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  HAVE_BOUND:sending write_to_hsm /pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/711/n13047113_0003_L010185R_D07_r4.cand.dogwood1.0.root to mover LTO4_109.mover ('131.225.13.205', 7869) DEL_DISM None
00:02:24 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  MSG_TYPE=ADD_TO_LMQ  request added to work at movers queue for storage group : minos
00:02:24 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  HAVE_BOUND:sending {'vc': {'storage_group': 'minos', 'external_label': 'VON262', 'library': 'CD-LTO4G1', 'file_family': 'minos', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'minos.minos.cpio_odc', 'file_family_width': 1}, 'outfilepath': '/pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/711/n13047113_0003_L010185R_D07_r4.cand.dogwood1.0.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 386616368L, 'mover': 'LTO4_109.mover', 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/711/n13047113_0003_L010185R_D07_r4.cand.dogwood1.0.root', 'uid': 0, 'uname': 'root', 'type': 'cpio_odc', 'gid': 0, 'mtime': 1257400942, 'rmajor': 0, 'machine': ('Linux', 'stkendca22a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (16893, 253994256L, 20L, 1, 42411, 5111, 512L, 1257400924, 1257400924, 1257381850), 'mode_octal': '0100036', 'mode': 33188, 'gname': 'root', 'size_bytes': 386616368L, 'fullname': '/diskb/write-pool-2/data/000F0000000000000B2472B8', 'inode': 254046904L, 'minor': 20}, 'version': 'v3_8b  CVS $Revision: 1.922 $ <frozen>', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskb/write-pool-2/data/000F0000000000000B2472B8', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/711/.(access)(000F0000000000000B2472B8)', 'fc': {'size': 386616368L, 'external_label': 'VON262', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/diskb/write-pool-2/data/000F0000000000000B2472B8', 'at_the_top': 13, 'local_inode': 1073749682L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.100', 40215), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257400944.3808351, 'encp_start_time': 1257400941.6235249, 'job_queued': 1257400942.5510621, 't0': 1257400941, 'in_queue': 1.8297119140625}, 'user_level': 'user2', 'unique_id': 'stkendca22a.fnal.gov-1257400942-18289-0'} to mover
00:02:24 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:02:24 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  WRITE_TO_HSM Thread MainThread
00:02:24 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  SENDING {'vc': {'storage_group': 'minos', 'external_label': 'VON262', 'library': 'CD-LTO4G1', 'file_family': 'minos', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'minos.minos.cpio_odc', 'file_family_width': 1}, 'outfilepath': '/pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/711/n13047113_0003_L010185R_D07_r4.cand.dogwood1.0.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 386616368L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7869, 'mc_device': '0,5,1,7', 'compression': 0, 'serial_num': '9310025229', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr209a', 'max_time_in_state': 1200, 'device': 'stkenmvr209a:/dev/rmt/tps0d0n', 'name': 'LTO4_109.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_109.stat', 'mover_address': ('131.225.13.205', 7869), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.205', 42023), 'data_ip': 'stkenmvr209a', 'logname': 'LTO4_109MV', 'hostip': '131.225.13.205', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/711/n13047113_0003_L010185R_D07_r4.cand.dogwood1.0.root', 'uid': 0, 'uname': 'root', 'mtime': 1257400942, 'rmajor': 0, 'machine': ('Linux', 'stkendca22a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16893, 253994256L, 20L, 1, 42411, 5111, 512L, 1257400924, 1257400924, 1257381850), 'mode_octal': '0100036', 'mode': 33188, 'size_bytes': 386616368L, 'gname': 'root', 'gid': 0, 'fullname': '/diskb/write-pool-2/data/000F0000000000000B2472B8', 'inode': 254046904L, 'minor': 20}, 'version': 'v3_8b  CVS $Revision: 1.922 $ <frozen>', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskb/write-pool-2/data/000F0000000000000B2472B8', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/711/.(access)(000F0000000000000B2472B8)', 'fc': {'size': 386616368L, 'external_label': 'VON262', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/diskb/write-pool-2/data/000F0000000000000B2472B8', 'at_the_top': 13, 'local_inode': 1073749682L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.100', 40215), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257400944.3808351, 'encp_start_time': 1257400941.6235249, 'job_queued': 1257400942.5510621, 't0': 1257400941, 'in_queue': 1.8297119140625}, 'user_level': 'user2', 'unique_id': 'stkendca22a.fnal.gov-1257400942-18289-0'} Thread client_connect_thread
00:02:24 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:02:24 stkendca22a.fnal.gov 018289 root I ENCP  Control socket ('131.225.13.100', 40215) is connected to ('131.225.13.205', 42024) for stkendca22a.fnal.gov-1257400942-18289-0.  elapsed=2.790sec
00:02:24 stkendca22a.fnal.gov 018289 root I ENCP  Received callback ticket from mover LTO4_109.mover for transfer stkendca22a.fnal.gov-1257400942-18289-0.  elapsed=2.790sec
00:02:24 stkendca22a.fnal.gov 018289 root I ENCP  Atempting to connect data socket to mover at ('131.225.13.205', 42023).
00:02:24 stkendca22a.fnal.gov 018289 root I ENCP  Connecting data socket to mover (('131.225.13.205', 42023)) with interface 131.225.13.100.  elapsed=2.790sec
00:02:24 stkendca22a.fnal.gov 018289 root I ENCP  Data socket ('131.225.13.100', 40218) is connected to ('131.225.13.205', 42023).  elapsed=2.790sec
00:02:24 stkendca22a.fnal.gov 018289 root I ENCP  Opening local file.
00:02:24 stkendca22a.fnal.gov 018289 root I ENCP  pf_status = ('ok', None)  skip_layer_cleanup = False
00:02:24 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  tape position: target 859 current 859 0 Thread seek_thread
00:02:24 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  write_tape starting, bytes_to_write=386616832 Thread tape_thread
00:02:24 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  Write starting. Tape VON262 absolute location in blocks 2540739 Thread tape_thread
00:02:25 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:02:27 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:02:27 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:02:28 stkendca22a.fnal.gov 018289 root I ENCP  File /diskb/write-pool-2/data/000F0000000000000B2472B8 transfered for stkendca22a.fnal.gov-1257400942-18289-0 in 3.68740820885 seconds.
00:02:29 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:02:31 southport.fnal.gov 000951 e898 I ENCP  Version: v3_6c  CVS $Revision: 1.822 $ <frozen>  OS: Linux 2.4.21-52.ELsmp i686 Release:  Scientific Linux release 3.0.5 (Fermi) LSB_VERSION="1.3" User: e898(3557)  Group: e898(5468)  Euser: e898(3557)  Egroup: e898(5468)  Library: CD-LTO4F1,CD-LTO4F1  Storage Group: miniboone  File Family: TankData  FF Wrapper: cpio_odc  FF Width: 1  Current working directory: southport.fnal.gov:/home/Nearline/admin  Command line: encp --delayed-dismount 1 /Nearline/data01/boone_0022362_0013.fc /pnfs/miniboone/TankData/run0022000_0022999/run0022360_0022369/boone_0022362_0013.fc
00:02:31 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  written bytes 386616832/386616832, blocks 2950 header 213 trailer 251 Thread tape_thread
00:02:31 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:02:33 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  filemarks written. Tape VON262 absolute location in blocks 2543690 Thread tape_thread
00:02:33 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  write_tape timing:pull 0.0328392982483 write 4.57219600677 crc_check 1.79902219772 freespace 0.0369040966034 block_write 6.44096159935 idle 0.0606429576874 Thread tape_thread
00:02:33 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  drive stats after write. Tape VON262 position 859 block 0 block_size 0 bloc_loc 2543690 tot_blocks 762939000 BOT 0 read_err 0 write_err 0 bytes 386616832 block_write_tot 6.44096159935 tape_rate 60024706.8759 Thread tape_thread
00:02:33 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  remaining bytes info in DB 464059392000 estimated 463672775168 reported from drive 463699968000 position 860 Thread tape_thread
00:02:33 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  new bitfile request {'uid': 0, 'external_label': 'VON262', 'gid': 0, 'location_cookie': '0000_000000000_0000859', 'complete_crc': 3407450939L, 'sanity_cookie': (65536L, 2288945019L), 'size': 386616368L} Thread tape_thread
00:02:33 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  set remaining: VON262 463699968000 0000_000000000_0000860 Thread tape_thread
00:02:33 last message repeated 1 times
00:02:33 stkensrv0n.fnal.gov 028762 enstore I VOLSRV  Updating  			update volume set last_access = '2009-11-05 00:02:33', eod_cookie = '0000_000000000_0000860', non_del_files = '862', remaining_bytes = 463699968000 where label = 'VON262';
00:02:33 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  transfer complete volume=VON262 location=860 Thread tape_thread
00:02:33 last message repeated 1 times
00:02:33 stkensrv0n.fnal.gov 028762 enstore I VOLSRV  Updating  			update volume set sum_wr_access = '859', modification_time = '2009-11-05 00:02:33' where label = 'VON262';
00:02:33 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  Sending done to client: {'vc': {'comment': '', 'storage_group': 'minos', 'blocksize': 131072, 'declared': 1248994044.0, 'library': 'CD-LTO4G1', 'si_time': [0.0, 1242939993.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VON262', 'wrapper': 'cpio_odc', 'remaining_bytes': 464059392000L, 'sum_mounts': 156, 'sum_rd_access': 0, 'file_family_width': 1, 'last_access': 1257400901.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0000859', 'capacity_bytes': 858993459200L, 'non_del_files': 861, 'sum_wr_err': 0, 'sum_wr_access': 858, 'file_family': 'minos', 'address': ('131.225.13.59', 7502), 'volume_family': 'minos.minos.cpio_odc', 'modification_time': 1257400901.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1257286976.0}, 'outfilepath': '/pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/711/n13047113_0003_L010185R_D07_r4.cand.dogwood1.0.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 386616368L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7869, 'mc_device': '0,5,1,7', 'compression': 0, 'serial_num': '9310025229', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr209a', 'max_time_in_state': 1200, 'device': 'stkenmvr209a:/dev/rmt/tps0d0n', 'name': 'LTO4_109.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_109.stat', 'mover_address': ('131.225.13.205', 7869), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.205', 42023), 'data_ip': 'stkenmvr209a', 'logname': 'LTO4_109MV', 'hostip': '131.225.13.205', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/711/n13047113_0003_L010185R_D07_r4.cand.dogwood1.0.root', 'uid': 0, 'uname': 'root', 'mtime': 1257400942, 'rmajor': 0, 'machine': ('Linux', 'stkendca22a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16893, 253994256L, 20L, 1, 42411, 5111, 512L, 1257400924, 1257400924, 1257381850), 'mode_octal': '0100036', 'mode': 33188, 'size_bytes': 386616368L, 'gname': 'root', 'gid': 0, 'fullname': '/diskb/write-pool-2/data/000F0000000000000B2472B8', 'inode': 254046904L, 'minor': 20}, 'version': 'v3_8b  CVS $Revision: 1.922 $ <frozen>', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskb/write-pool-2/data/000F0000000000000B2472B8', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/711/.(access)(000F0000000000000B2472B8)', 'fc': {'uid': 0, 'external_label': 'VON262', 'gid': 0, 'location_cookie': '0000_000000000_0000859', 'complete_crc': 3407450939L, 'bfid': 'CDMS125740095300000', 'sanity_cookie': (65536L, 2288945019L), 'size': 386616368L}, 'infilepath': '/diskb/write-pool-2/data/000F0000000000000B2472B8', 'at_the_top': 13, 'local_inode': 1073749682L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.100', 40215), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257400941.6235249, 'seek_time': 0.029752016067504883, 'transfer_time': 8.8858118057250977, 't0': 1257400941, 'mount_time': 0.019254207611083984, 'lm_dequeued': 1257400944.3808351, 'job_queued': 1257400942.5510621, 'drive_call_time': 4.5293986797332764, 'in_queue': 1.8297119140625, 'drive_transfer_time': 8.0232284069061279}, 'user_level': 'user2', 'unique_i(trunc. 4126) Thread tape_thread
00:02:33 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  Done is sent Thread tape_thread
00:02:33 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  maybe_clean Thread tape_thread
00:02:33 stkendca22a.fnal.gov 018289 root I ENCP  Received final dialog for stkendca22a.fnal.gov-1257400942-18289-0.  elapsed=11.760sec
00:02:33 stkendca22a.fnal.gov 018289 root I ENCP  leaving transfer_file().
00:02:33 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  returned from maybe_clean Thread tape_thread
00:02:33 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:02:33 stkendca22a.fnal.gov 018289 root I ENCP  pf_status = ('ok', None)  skip_layer_cleanup = False
00:02:33 stkensrv0n.fnal.gov 028796 enstore I FILSRV  Updating  			update file set deleted = 'n', pnfs_path = '/pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/711/n13047113_0003_L010185R_D07_r4.cand.dogwood1.0.root', drive = 'stkenmvr209a:/dev/rmt/tps0d0n:9310025229', pnfs_id = '000F0000000000000B2472B8' where bfid = 'CDMS125740095300000';
00:02:33 stkendca22a.fnal.gov 018289 root I ENCP  MSG_TYPE=ENCP_XFER    write_to_hsm /diskb/write-pool-2/data/000F0000000000000B2472B8 -> /pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/711/n13047113_0003_L010185R_D07_r4.cand.dogwood1.0.root: 386616368 bytes copied to VON262 at 41.4 MB/S (90.7 MB/S network) (46 MB/S drive) (100 MB/S disk) mover=LTO4_109.mover drive_id=ULTRIUM-TD4 drive_sn=9310025229 drive_vendor=IBM elapsed=12.011 {'storage_group': 'minos', 'encp_crc': 3407450939L, 'disk_rate': 100.11251963080535, 'drive_rate': 45.954830370589633, 'driver': 'FTTDriver', 'transfer_rate': 41.434052435381332, 'mover_interface': 'stkenmvr209a', 'overall_rate': 39.990691438330622, 'media_changer': 'SL8500G1.media_changer', 'network_rate': 41.434052435381332, 'encp_ip': '131.225.13.100', 'unique_id': 'stkendca22a.fnal.gov-1257400942-18289-0'}
00:02:33 stkendca22a.fnal.gov 018289 root I ENCP  File /pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/711/n13047113_0003_L010185R_D07_r4.cand.dogwood1.0.root copied successfully.
00:02:33 stkendca22a.fnal.gov 018289 root I ENCP  Completed transferring 386616368 bytes in 1 files in 12.0115840435 sec.  Overall rate = 40 MB/sec.  Transfer rate = 41.4 MB/sec.  Network rate = 90.7 MB/sec.  Drive rate = 46 MB/sec.  Disk rate = 100 MB/sec.  Exit status = 0.
00:02:33 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:02:35 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:02:35 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:02:36 southport.fnal.gov 000951 e898 I ENCP  Sending /pnfs/miniboone/TankData/run0022000_0022999/run0022360_0022369/boone_0022362_0013.fc write request to LM: uninque_id: southport.fnal.gov-1257400951-951-0 inputfile: /Nearline/data01/boone_0022362_0013.fc outputfile: /pnfs/miniboone/TankData/run0022000_0022999/run0022360_0022369/boone_0022362_0013.fc
00:02:36 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  MSG_TYPE=ADD_TO_LMQ  request added to pending queue for storage group : miniboone
00:02:36 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  write Q'd /Nearline/data01/boone_0022362_0013.fc (southport.fnal.gov-1257400951-951-0) -> /pnfs/miniboone/TankData/run0022000_0022999/run0022360_0022369/boone_0022362_0013.fc : library=CD-LTO4F1 family=TankData requester:e898 volume_family:miniboone.TankData.cpio_odc
00:02:36 southport.fnal.gov 000951 e898 I ENCP  Waiting for mover to connect control socket at address: ('131.225.194.144', 55692)  elapsed=5.310sec
00:02:37 stkensrv0n.fnal.gov 028762 enstore I VOLSRV  Assigning  volume VOO096 from storage group miniboone.TankData.cpio_odc to library CD-LTO4F1, volume family miniboone.TankData.cpio_odc
00:02:37 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  write_to_hsm work on vol=VOO096 mover=LTO4_117.mover requester:e898
00:02:37 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  MSG_TYPE=ADD_TO_LMQ  request added to work at movers queue for storage group : miniboone
00:02:37 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  IDLE:sending {'vc': {'storage_group': 'miniboone', 'wrapper': 'cpio_odc', 'file_family': 'TankData', 'library': 'CD-LTO4F1', 'address': ('131.225.13.59', 7502), 'volume_family': 'miniboone.TankData.cpio_odc', 'file_family_width': 1}, 'encp': {'agetime': 30, 'adminpri': 1, 'delpri': 1, 'basepri': 1000003, 'delayed_dismount': 1, 'curpri': 1000003}, 'file_size': 104858172L, 'mover': 'LTO4_117.mover', 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8542)}, 'wrapper': {'major': 0, 'mode_octal': '0100011', 'pnfsFilename': '/pnfs/miniboone/TankData/run0022000_0022999/run0022360_0022369/boone_0022362_0013.fc', 'uid': 3557, 'uname': 'e898', 'type': 'cpio_odc', 'mtime': 1257400951, 'rmajor': 0, 'machine': ('Linux', 'southport.fnal.gov', '2.4.21-52.ELsmp', '#1 SMP Fri Sep 28 13:31:57 CDT 2007', 'i686'), 'sanity_size': 65536, 'pstat': (16872, 884208856L, 10L, 1, 3557, 5468, 512L, 1257399951, 1257399951, 1257354573), 'rminor': 0, 'mode': 33188, 'size_bytes': 104858172L, 'gname': 'e898', 'gid': 5468, 'fullname': '/Nearline/data01/boone_0022362_0013.fc', 'inode': 884210656L, 'minor': 10}, 'version': 'v3_6c  CVS $Revision: 1.822 $ <frozen>', 'encp_daq': None, 'client_crc': 1, 'infile': '/Nearline/data01/boone_0022362_0013.fc', 'status': ('ok', None), 'outfile': '/pnfs/miniboone/TankData/run0022000_0022999/run0022360_0022369/boone_0022362_0013.fc', 'fc': {'size': 104858172L, 'external_label': 'VOO096', 'pnfsid': '003400000000000000B3FBE0', 'address': ('131.225.13.59', 7501)}, 'at_the_top': 2, 'local_inode': 3621L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.194.144', 55692), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257400957.1516631, 'encp_start_time': 1257400951.163584, 'job_queued': 1257400956.4348819, 't0': 1257400951, 'in_queue': 0.71673798561096191}, 'unique_id': 'southport.fnal.gov-1257400951-951-0'} to mover
00:02:37 stkenmvr217a.fnal.gov 007075 root I LTO4_117MV  WRITE_TO_HSM Thread MainThread
00:02:37 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  can not remove from sg LTO4_117.mover VOO096
00:02:37 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  SG: {'miniboone': [('LTO4_115.mover', 'VOO437')]}
00:02:37 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  can not remove from vf LTO4_117.mover VOO096
00:02:37 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  VF: {'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]}
00:02:37 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437'), ('LTO4_117.mover', 'VOO096')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:02:37 last message repeated 1 times
00:02:37 stkenmvr217a.fnal.gov 007075 root I LTO4_117MV  SENDING {'vc': {'storage_group': 'miniboone', 'wrapper': 'cpio_odc', 'file_family': 'TankData', 'library': 'CD-LTO4F1', 'address': ('131.225.13.59', 7502), 'volume_family': 'miniboone.TankData.cpio_odc', 'file_family_width': 1}, 'encp': {'agetime': 30, 'adminpri': 1, 'delpri': 1, 'basepri': 1000003, 'delayed_dismount': 1, 'curpri': 1000003}, 'file_size': 104858172L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 50, 'max_failures': 5, 'library': 'CD-LTO4F1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7877, 'mc_device': '1,2,1,8', 'compression': 0, 'serial_num': '1310151225', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr217a', 'max_time_in_state': 1200, 'device': 'stkenmvr217a:/dev/rmt/tps0d0n', 'name': 'LTO4_117.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_117.stat', 'mover_address': ('131.225.13.113', 7877), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.113', 33066), 'data_ip': 'stkenmvr217a', 'logname': 'LTO4_117MV', 'hostip': '131.225.13.113', 'do_cleaning': 'No', 'media_changer': 'SL8500F1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8542)}, 'wrapper': {'major': 0, 'mode_octal': '0100011', 'pnfsFilename': '/pnfs/miniboone/TankData/run0022000_0022999/run0022360_0022369/boone_0022362_0013.fc', 'uid': 3557, 'uname': 'e898', 'type': 'cpio_odc', 'rmajor': 0, 'rminor': 0, 'machine': ('Linux', 'southport.fnal.gov', '2.4.21-52.ELsmp', '#1 SMP Fri Sep 28 13:31:57 CDT 2007', 'i686'), 'sanity_size': 65536, 'pstat': (16872, 884208856L, 10L, 1, 3557, 5468, 512L, 1257399951, 1257399951, 1257354573), 'gid': 5468, 'mtime': 1257400951, 'mode': 33188, 'gname': 'e898', 'size_bytes': 104858172L, 'fullname': '/Nearline/data01/boone_0022362_0013.fc', 'inode': 884210656L, 'minor': 10}, 'version': 'v3_6c  CVS $Revision: 1.822 $ <frozen>', 'encp_daq': None, 'client_crc': 1, 'infile': '/Nearline/data01/boone_0022362_0013.fc', 'status': ('ok', None), 'outfile': '/pnfs/miniboone/TankData/run0022000_0022999/run0022360_0022369/boone_0022362_0013.fc', 'fc': {'size': 104858172L, 'external_label': 'VOO096', 'pnfsid': '003400000000000000B3FBE0', 'address': ('131.225.13.59', 7501)}, 'at_the_top': 2, 'local_inode': 3621L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.194.144', 55692), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257400957.1516631, 'encp_start_time': 1257400951.163584, 'job_queued': 1257400956.4348819, 't0': 1257400951, 'in_queue': 0.71673798561096191}, 'unique_id': 'southport.fnal.gov-1257400951-951-0'} Thread client_connect_thread
00:02:37 southport.fnal.gov 000951 e898 I ENCP  Control socket ('131.225.194.144', 55692) is connected to ('131.225.13.113', 33067) for southport.fnal.gov-1257400951-951-0.  elapsed=6.040sec
00:02:37 southport.fnal.gov 000951 e898 I ENCP  Received callback ticket from mover LTO4_117.mover for transfer southport.fnal.gov-1257400951-951-0.  elapsed=6.040sec
00:02:37 southport.fnal.gov 000951 e898 I ENCP  Atempting to connect data socket to mover at ('131.225.13.113', 33066).
00:02:37 southport.fnal.gov 000951 e898 I ENCP  Connecting data socket to mover (('131.225.13.113', 33066)) with interface 131.225.194.144.  elapsed=6.050sec
00:02:37 southport.fnal.gov 000951 e898 I ENCP  Data socket ('131.225.194.144', 55694) is connected to ('131.225.13.113', 33066).  elapsed=6.050sec
00:02:37 stkenmvr217a.fnal.gov 007075 root I LTO4_117MV  MSG_TYPE=MC_LOAD_REQ  mounting VOO096 ULTRIUM-TD4 00:02:37 Thread media_thread
00:02:37 stkensrv4n.fnal.gov 005843 enstore I SL8500F1MC  PROCESSING mount VOO096 1,2,1,8
00:02:37 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:02:37 southport.fnal.gov 000951 e898 I ENCP  pf_status = ('ok', None)  skip_layer_cleanup = 0
00:02:37 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:02:39 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:02:39 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:02:41 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:02:41 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:02:43 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:02:43 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:02:45 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:02:45 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:02:47 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:02:47 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:02:48 stkensrv4n.fnal.gov 007028 enstore I SL8500F1MC  query drive 1,2,1,8 => 0,1,2,1,8 online          available              IBM-LTO4
00:02:48 stkensrv4n.fnal.gov 007028 enstore I SL8500F1MC  Dismount 14 ignored: Drive 1,2,1,8 is empty. Thought VOO096 was there =>. dismount VOLUME 1,2,1,8 force => ok,1,2,1,8 online          available              IBM-LTO4
00:02:49 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:02:49 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:02:51 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:02:51 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:02:53 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437'), ('LTO4_117.mover', 'VOO096')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:02:53 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:02:53 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:02:53 stkensrv4n.fnal.gov 005357 enstore E LTO4G1LM  MOVER ERROR RQ {'status': ('ok', None), 'returned_work': None, 'library': None, 'mover_type': '', 'current_location': '0000_000000000_0000000', 'ip_map': '', 'operation': None, 'read_only': 0, 'current_time': 1257400973.8671319, 'error_source': None, 'external_label': None, 'state': 'OFFLINE', 'r_a': (('131.225.13.194', 54880), 16281L, '131.225.13.194-54880-1256586929.509013-4866'), 'library_list': [('CD-LTO4G1.library_manager', ('131.225.13.4', 8541))], 'current_priority': (None, None), 'transfer_deficiency': 1, 'time_in_state': 814043.50707387924, 'address': ('131.225.13.194', 7860), 'volume_family': None, 'work': 'mover_error', 'mover': 'LTO4_99.mover', 'volume_status': (['none', 'none'], ['none', 'none']), 'unique_id': None}
00:02:55 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:02:55 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:02:57 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:02:57 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:02:58 stkensrv4n.fnal.gov 007028 enstore I SL8500F1MC  query vol VOO096 => 0,VOO096      home              1, 1,45,20, 1         LTO-800G
00:02:59 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:02:59 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:03:01 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:03:01 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:03:03 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:03:03 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:03:05 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:03:05 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:03:07 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:03:07 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:03:08 stkensrv4n.fnal.gov 007028 enstore I SL8500F1MC  query drive 1,2,1,8 => 0,1,2,1,8 online          available              IBM-LTO4
00:03:09 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:03:09 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:03:11 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:03:11 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:03:13 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:03:13 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:03:15 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:03:15 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:03:17 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:03:17 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:03:19 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:03:19 last message repeated 1 times
00:03:19 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:03:21 stkendca18a.fnal.gov 021252 root I ENCP  File /diska/write-pool-1/data/0030000000000000061EB170 transfered for stkendca18a.fnal.gov-1257400876-21252-0 in 88.2248837948 seconds.
00:03:21 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:03:21 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:03:21 stkendca22a.fnal.gov 019626 root I ENCP  Version: v3_8b  CVS $Revision: 1.922 $ <frozen>  OS: Linux 2.6.9-89.0.9.ELsmp i686 Release:  Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0)  Group: root(0)  Euser: root(0)  Egroup: root(0)  Library: garbage  Storage Group: test  File Family: dcache  FF Wrapper: cpio_odc  FF Width: 1  Current working directory: stkendca22a.fnal.gov:/home/enstore/dcache-log  Command line: /fnal/ups/prd/encp/v3_8b/Linux-2-6-dcache/encp --verbose 4 --threaded --ecrc --bypass-filesystem-max-filesize-check --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0003 --put-cache 0005000000000000005FA5F8 /diskb/write-pool-2/data/0005000000000000005FA5F8
00:03:21 stkendca22a.fnal.gov 019627 root I ENCP  Version: v3_8b  CVS $Revision: 1.922 $ <frozen>  OS: Linux 2.6.9-89.0.9.ELsmp i686 Release:  Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0)  Group: root(0)  Euser: root(0)  Egroup: root(0)  Library: garbage  Storage Group: test  File Family: dcache  FF Wrapper: cpio_odc  FF Width: 1  Current working directory: stkendca22a.fnal.gov:/home/enstore/dcache-log  Command line: /fnal/ups/prd/encp/v3_8b/Linux-2-6-dcache/encp --verbose 4 --threaded --ecrc --bypass-filesystem-max-filesize-check --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0001 --put-cache 0005000000000000005FA590 /diskb/write-pool-2/data/0005000000000000005FA590
00:03:21 stkendca22a.fnal.gov 019639 root I ENCP  Version: v3_8b  CVS $Revision: 1.922 $ <frozen>  OS: Linux 2.6.9-89.0.9.ELsmp i686 Release:  Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0)  Group: root(0)  Euser: root(0)  Egroup: root(0)  Library: garbage  Storage Group: test  File Family: dcache  FF Wrapper: cpio_odc  FF Width: 1  Current working directory: stkendca22a.fnal.gov:/home/enstore/dcache-log  Command line: /fnal/ups/prd/encp/v3_8b/Linux-2-6-dcache/encp --verbose 4 --threaded --ecrc --bypass-filesystem-max-filesize-check --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0002 --put-cache 0005000000000000005FA5C8 /diskb/write-pool-2/data/0005000000000000005FA5C8
00:03:22 stkendca22a.fnal.gov 019626 root I ENCP  Configuration Server: no such name: 'garbage.library_manager'
00:03:22 stkendca22a.fnal.gov 019626 root E ENCP  INFILE=/diskb/write-pool-2/data/0005000000000000005FA5F8 OUTFILE=/pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0003 FILESIZE=801512 LABEL= LOCATION= DRIVE= DRIVE_SN= TRANSFER_TIME=0.00 SEEK_TIME=0.00 MOUNT_TIME=0.00 QWAIT_TIME=0.00 TIME2NOW=0.88 CRC= STATUS=KEYERROR Configuration Server: no such name: 'garbage.library_manager'
00:03:22 stkendca22a.fnal.gov 019627 root I ENCP  Configuration Server: no such name: 'garbage.library_manager'
00:03:22 stkendca22a.fnal.gov 019627 root E ENCP  INFILE=/diskb/write-pool-2/data/0005000000000000005FA590 OUTFILE=/pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0001 FILESIZE=801512 LABEL= LOCATION= DRIVE= DRIVE_SN= TRANSFER_TIME=0.00 SEEK_TIME=0.00 MOUNT_TIME=0.00 QWAIT_TIME=0.00 TIME2NOW=0.91 CRC= STATUS=KEYERROR Configuration Server: no such name: 'garbage.library_manager'
00:03:22 stkendca22a.fnal.gov 019639 root I ENCP  Configuration Server: no such name: 'garbage.library_manager'
00:03:22 stkendca22a.fnal.gov 019639 root E ENCP  INFILE=/diskb/write-pool-2/data/0005000000000000005FA5C8 OUTFILE=/pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0002 FILESIZE=801512 LABEL= LOCATION= DRIVE= DRIVE_SN= TRANSFER_TIME=0.00 SEEK_TIME=0.00 MOUNT_TIME=0.00 QWAIT_TIME=0.00 TIME2NOW=0.88 CRC= STATUS=KEYERROR Configuration Server: no such name: 'garbage.library_manager'
00:03:23 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:03:23 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:03:25 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  written bytes 1654612480/1654612480, blocks 12624 header 189 trailer 323 Thread tape_thread
00:03:25 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:03:25 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:03:26 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  filemarks written. Tape VON743 absolute location in blocks 1936087 Thread tape_thread
00:03:26 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  write_tape timing:pull 0.149878501892 write 7.67864513397 crc_check 6.45105171204 freespace 0.106536388397 block_write 14.3861117363 idle 0.199841022491 Thread tape_thread
00:03:26 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  drive stats after write. Tape VON743 position 133 block 0 block_size 0 bloc_loc 1936087 tot_blocks 762939000 BOT 0 read_err 0 write_err 0 bytes 1654612480 block_write_tot 14.3861117363 tape_rate 115014571.715 Thread tape_thread
00:03:26 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  remaining bytes info in DB 538925056000 estimated 537270443520 reported from drive 537370624000 position 134 Thread tape_thread
00:03:26 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  new bitfile request {'uid': 0, 'external_label': 'VON743', 'gid': 0, 'location_cookie': '0000_000000000_0000133', 'complete_crc': 2675388236L, 'sanity_cookie': (65536L, 29196594L), 'size': 1654611968L} Thread tape_thread
00:03:26 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  set remaining: VON743 537370624000 0000_000000000_0000134 Thread tape_thread
00:03:26 last message repeated 1 times
00:03:26 stkensrv0n.fnal.gov 028762 enstore I VOLSRV  Updating  			update volume set last_access = '2009-11-05 00:03:26', eod_cookie = '0000_000000000_0000134', non_del_files = '135', remaining_bytes = 537370624000 where label = 'VON743';
00:03:26 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  transfer complete volume=VON743 location=134 Thread tape_thread
00:03:26 last message repeated 1 times
00:03:26 stkensrv0n.fnal.gov 028762 enstore I VOLSRV  Updating  			update volume set sum_wr_access = '133', modification_time = '2009-11-05 00:03:26' where label = 'VON743';
00:03:26 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  Sending done to client: {'vc': {'comment': '', 'storage_group': 'exp-db', 'blocksize': 131072, 'declared': 1256853181.0, 'library': 'CD-LTO4G1', 'si_time': [0.0, 1255497368.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VON743', 'wrapper': 'cpio_odc', 'remaining_bytes': 538925056000L, 'sum_mounts': 46, 'sum_rd_access': 0, 'file_family_width': 2, 'last_access': 1257400911.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0000133', 'capacity_bytes': 858993459200L, 'non_del_files': 134, 'sum_wr_err': 0, 'sum_wr_access': 132, 'file_family': 'daily-d0-offline', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.daily-d0-offline.cpio_odc', 'modification_time': 1257400911.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1257340831.0}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702070562_s103266_p25', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 1654611968L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'dismount_delay': 30, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7842, 'mc_device': '0,7,1,12', 'compression': 0, 'serial_num': '1310155269', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr142a', 'max_time_in_state': 1200, 'device': 'stkenmvr142a:/dev/rmt/tps0d0n', 'name': 'LTO4_42.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_42.stat', 'single_filemark': 1, 'mount_delay': 15, 'mover_address': ('131.225.13.170', 7842), 'check_first_written_file': 0, 'callback_addr': ('131.225.13.170', 52230), 'data_ip': 'stkenmvr142a', 'logname': 'LTO4_42MV', 'hostip': '131.225.13.170', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer', 'min_buffer': 67108864L}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702070562_s103266_p25', 'uid': 0, 'uname': 'root', 'mtime': 1257400876, 'rmajor': 0, 'machine': ('Linux', 'stkendca18a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 820619016L, 20L, 1, 1602, 2752, 512L, 1257400848, 1257400848, 1254398418), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 1654611968L, 'gname': 'root', 'gid': 0, 'fullname': '/diska/write-pool-1/data/0030000000000000061EB170', 'inode': 807317872L, 'minor': 20}, 'version': 'v3_8b  CVS $Revision: 1.922 $ <frozen>', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diska/write-pool-1/data/0030000000000000061EB170', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/.(access)(0030000000000000061EB170)', 'fc': {'uid': 0, 'external_label': 'VON743', 'gid': 0, 'location_cookie': '0000_000000000_0000133', 'complete_crc': 2675388236L, 'bfid': 'CDMS125740100600000', 'sanity_cookie': (65536L, 29196594L), 'size': 1654611968L}, 'infilepath': '/diska/write-pool-1/data/0030000000000000061EB170', 'at_the_top': 339, 'local_inode': 622464081L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.6', 54459), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257400874.723093, 'seek_time': 0.034677028656005859, 'transfer_time': 94.081488132476807, 't0': 1257400874, 'mount_time': 0.041721820831298828, 'lm_dequeued': 1257400912.722043, 'job_queued': 1257400876.4799681, 'drive_call_time': 7.5270218849182129, 'in_queue': 36.241796970367432, 'drive_transfer_time': 16.283878564834595}, 'user_level': 'us(trunc. 4141) Thread tape_thread
00:03:26 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  Done is sent Thread tape_thread
00:03:26 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  maybe_clean Thread tape_thread
00:03:26 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  returned from maybe_clean Thread tape_thread
00:03:26 stkendca18a.fnal.gov 021252 root I ENCP  Received final dialog for stkendca18a.fnal.gov-1257400876-21252-0.  elapsed=132.260sec
00:03:26 stkendca18a.fnal.gov 021252 root I ENCP  leaving transfer_file().
00:03:27 stkendca18a.fnal.gov 021252 root I ENCP  pf_status = ('ok', None)  skip_layer_cleanup = False
00:03:27 stkensrv0n.fnal.gov 028796 enstore I FILSRV  Updating  			update file set deleted = 'n', pnfs_path = '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702070562_s103266_p25', drive = 'stkenmvr142a:/dev/rmt/tps0d0n:1310155269', pnfs_id = '0030000000000000061EB170' where bfid = 'CDMS125740100600000';
00:03:27 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:03:27 stkendca18a.fnal.gov 021252 root I ENCP  MSG_TYPE=ENCP_XFER    write_to_hsm /diska/write-pool-1/data/0030000000000000061EB170 -> /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702070562_s103266_p25: 1654611968 bytes copied to VON743 at 16.8 MB/S (66.2 MB/S network) (96.9 MB/S drive) (17.9 MB/S disk) mover=LTO4_42.mover drive_id=ULTRIUM-TD4 drive_sn=1310155269 drive_vendor=IBM elapsed=132.74 {'storage_group': 'exp-db', 'encp_crc': 2675388236L, 'disk_rate': 17.913782751536399, 'drive_rate': 96.903261174376624, 'driver': 'FTTDriver', 'transfer_rate': 16.76886508443225, 'mover_interface': 'stkenmvr142a', 'overall_rate': 16.666008025340972, 'media_changer': 'SL8500G1.media_changer', 'network_rate': 16.76886508443225, 'encp_ip': '131.225.13.6', 'unique_id': 'stkendca18a.fnal.gov-1257400876-21252-0'}
00:03:27 stkendca18a.fnal.gov 021252 root I ENCP  File /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702070562_s103266_p25 copied successfully.
00:03:27 stkendca18a.fnal.gov 021252 root I ENCP  Completed transferring 1654611968 bytes in 1 files in 132.753371 sec.  Overall rate = 16.7 MB/sec.  Transfer rate = 16.8 MB/sec.  Network rate = 66.2 MB/sec.  Drive rate = 96.9 MB/sec.  Disk rate = 17.9 MB/sec.  Exit status = 0.
00:03:27 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:03:27 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  write_to_hsm next work on vol=VON743 mover=LTO4_42.mover requester:root
00:03:27 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  HAVE_BOUND:sending write_to_hsm /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702068213_s103263_p3 to mover LTO4_42.mover ('131.225.13.170', 7842) DEL_DISM None
00:03:27 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  MSG_TYPE=ADD_TO_LMQ  request added to work at movers queue for storage group : exp-db
00:03:27 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  HAVE_BOUND:sending {'vc': {'storage_group': 'exp-db', 'external_label': 'VON743', 'library': 'CD-LTO4G1', 'file_family': 'daily-d0-offline', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.daily-d0-offline.cpio_odc', 'file_family_width': 2}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702068213_s103263_p3', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 1966080000L, 'mover': 'LTO4_42.mover', 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702068213_s103263_p3', 'uid': 0, 'uname': 'root', 'type': 'cpio_odc', 'gid': 0, 'mtime': 1257400927, 'rmajor': 0, 'machine': ('Linux', 'stkendca18a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (16877, 820618800L, 20L, 1, 1602, 2752, 512L, 1257400896, 1257400896, 1254398418), 'mode_octal': '0100016', 'mode': 33188, 'gname': 'root', 'size_bytes': 1966080000L, 'fullname': '/diskb/write-pool-2/data/0030000000000000061EB018', 'inode': 807317528L, 'minor': 20}, 'version': 'v3_8b  CVS $Revision: 1.922 $ <frozen>', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskb/write-pool-2/data/0030000000000000061EB018', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/.(access)(0030000000000000061EB018)', 'fc': {'size': 1966080000L, 'external_label': 'VON743', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/diskb/write-pool-2/data/0030000000000000061EB018', 'at_the_top': 1184, 'local_inode': 536873447L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.6', 54468), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257401007.804265, 'encp_start_time': 1257400926.3102441, 'job_queued': 1257400927.840569, 't0': 1257400926, 'in_queue': 79.963407039642334}, 'user_level': 'user2', 'unique_id': 'stkendca18a.fnal.gov-1257400927-22164-0'} to mover
00:03:27 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:03:27 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  WRITE_TO_HSM Thread MainThread
00:03:27 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  SENDING {'vc': {'storage_group': 'exp-db', 'external_label': 'VON743', 'library': 'CD-LTO4G1', 'file_family': 'daily-d0-offline', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.daily-d0-offline.cpio_odc', 'file_family_width': 2}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702068213_s103263_p3', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 1966080000L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'dismount_delay': 30, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7842, 'mc_device': '0,7,1,12', 'compression': 0, 'serial_num': '1310155269', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr142a', 'max_time_in_state': 1200, 'device': 'stkenmvr142a:/dev/rmt/tps0d0n', 'name': 'LTO4_42.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_42.stat', 'single_filemark': 1, 'mount_delay': 15, 'mover_address': ('131.225.13.170', 7842), 'check_first_written_file': 0, 'callback_addr': ('131.225.13.170', 52233), 'data_ip': 'stkenmvr142a', 'logname': 'LTO4_42MV', 'hostip': '131.225.13.170', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer', 'min_buffer': 67108864L}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702068213_s103263_p3', 'uid': 0, 'uname': 'root', 'mtime': 1257400927, 'rmajor': 0, 'machine': ('Linux', 'stkendca18a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 820618800L, 20L, 1, 1602, 2752, 512L, 1257400896, 1257400896, 1254398418), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 1966080000L, 'gname': 'root', 'gid': 0, 'fullname': '/diskb/write-pool-2/data/0030000000000000061EB018', 'inode': 807317528L, 'minor': 20}, 'version': 'v3_8b  CVS $Revision: 1.922 $ <frozen>', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskb/write-pool-2/data/0030000000000000061EB018', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/.(access)(0030000000000000061EB018)', 'fc': {'size': 1966080000L, 'external_label': 'VON743', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/diskb/write-pool-2/data/0030000000000000061EB018', 'at_the_top': 1184, 'local_inode': 536873447L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.6', 54468), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257401007.804265, 'encp_start_time': 1257400926.3102441, 'job_queued': 1257400927.840569, 't0': 1257400926, 'in_queue': 79.963407039642334}, 'user_level': 'user2', 'unique_id': 'stkendca18a.fnal.gov-1257400927-22164-0'} Thread client_connect_thread
00:03:27 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:03:27 stkendca18a.fnal.gov 022164 root I ENCP  Control socket ('131.225.13.6', 54468) is connected to ('131.225.13.170', 52234) for stkendca18a.fnal.gov-1257400927-22164-0.  elapsed=81.550sec
00:03:27 stkendca18a.fnal.gov 022164 root I ENCP  Received callback ticket from mover LTO4_42.mover for transfer stkendca18a.fnal.gov-1257400927-22164-0.  elapsed=81.550sec
00:03:27 stkendca18a.fnal.gov 022164 root I ENCP  Atempting to connect data socket to mover at ('131.225.13.170', 52233).
00:03:27 stkendca18a.fnal.gov 022164 root I ENCP  Connecting data socket to mover (('131.225.13.170', 52233)) with interface 131.225.13.6.  elapsed=81.550sec
00:03:27 stkendca18a.fnal.gov 022164 root I ENCP  Data socket ('131.225.13.6', 54473) is connected to ('131.225.13.170', 52233).  elapsed=81.560sec
00:03:27 stkendca18a.fnal.gov 022164 root I ENCP  Opening local file.
00:03:27 stkendca18a.fnal.gov 022164 root I ENCP  pf_status = ('ok', None)  skip_layer_cleanup = False
00:03:27 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  tape position: target 134 current 134 0 Thread seek_thread
00:03:27 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  write_tape starting, bytes_to_write=1966080512 Thread tape_thread
00:03:27 stkenmvr142a.fnal.gov 009485 root I LTO4_42MV  Write starting. Tape VON743 absolute location in blocks 1936087 Thread tape_thread
00:03:28 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:03:29 last message repeated 1 times
00:03:29 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:03:31 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:03:31 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:03:33 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:03:33 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  Updating stats Thread media_thread
00:03:33 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  writing a tape mark before dismount Thread media_thread
00:03:33 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:03:36 last message repeated 1 times
00:03:36 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  Ejecting tape Thread media_thread
00:03:37 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:03:41 last message repeated 1 times
00:03:41 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437'), ('LTO4_117.mover', 'VOO096')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:03:41 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:03:43 last message repeated 1 times
00:03:43 stkensrv4n.fnal.gov 005357 enstore E LTO4G1LM  MOVER ERROR RQ {'status': ('ok', None), 'returned_work': None, 'library': None, 'mover_type': '', 'current_location': '0000_000000000_0000000', 'ip_map': '', 'operation': None, 'read_only': 0, 'current_time': 1257401023.8714311, 'error_source': None, 'external_label': None, 'state': 'OFFLINE', 'r_a': (('131.225.13.194', 54880), 16282L, '131.225.13.194-54880-1256586929.509013-4866'), 'library_list': [('CD-LTO4G1.library_manager', ('131.225.13.4', 8541))], 'current_priority': (None, None), 'transfer_deficiency': 1, 'time_in_state': 814093.51137304306, 'address': ('131.225.13.194', 7860), 'volume_family': None, 'work': 'mover_error', 'mover': 'LTO4_99.mover', 'volume_status': (['none', 'none'], ['none', 'none']), 'unique_id': None}
00:03:45 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:03:46 stkendca18a.fnal.gov 021257 root I ENCP  File /diska/write-pool-1/data/0030000000000000061EB148 transfered for stkendca18a.fnal.gov-1257400876-21257-0 in 107.726224899 seconds.
00:03:47 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:03:48 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  written bytes 1966080512/1966080512, blocks 15001 header 188 trailer 324 Thread tape_thread
00:03:49 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:03:50 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  filemarks written. Tape VON266 absolute location in blocks 2649429 Thread tape_thread
00:03:50 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  write_tape timing:pull 0.188437461853 write 7.89233732224 crc_check 8.99346637726 freespace 0.136174440384 block_write 17.2104156017 idle 0.252036333084 Thread tape_thread
00:03:50 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  drive stats after write. Tape VON266 position 187 block 0 block_size 0 bloc_loc 2649429 tot_blocks 762939000 BOT 0 read_err 0 write_err 0 bytes 1966080512 block_write_tot 17.2104156017 tape_rate 114237828.853 Thread tape_thread
00:03:50 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  remaining bytes info in DB 452280320000 estimated 450314239488 reported from drive 450447360000 position 188 Thread tape_thread
00:03:50 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  new bitfile request {'uid': 0, 'external_label': 'VON266', 'gid': 0, 'location_cookie': '0000_000000000_0000187', 'complete_crc': 782989186L, 'sanity_cookie': (65536L, 4162356723L), 'size': 1966080000L} Thread tape_thread
00:03:50 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  set remaining: VON266 450447360000 0000_000000000_0000188 Thread tape_thread
00:03:50 last message repeated 1 times
00:03:50 stkensrv0n.fnal.gov 028762 enstore I VOLSRV  Updating  			update volume set last_access = '2009-11-05 00:03:50', eod_cookie = '0000_000000000_0000188', non_del_files = '188', remaining_bytes = 450447360000 where label = 'VON266';
00:03:50 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  transfer complete volume=VON266 location=188 Thread tape_thread
00:03:50 last message repeated 1 times
00:03:50 stkensrv0n.fnal.gov 028762 enstore I VOLSRV  Updating  			update volume set sum_wr_access = '187', modification_time = '2009-11-05 00:03:50' where label = 'VON266';
00:03:50 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  Sending done to client: {'vc': {'comment': '', 'storage_group': 'exp-db', 'blocksize': 131072, 'declared': 1248994048.0, 'library': 'CD-LTO4G1', 'si_time': [0.0, 1242744483.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VON266', 'wrapper': 'cpio_odc', 'remaining_bytes': 452280320000L, 'sum_mounts': 73, 'sum_rd_access': 0, 'file_family_width': 2, 'last_access': 1257400918.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0000187', 'capacity_bytes': 858993459200L, 'non_del_files': 187, 'sum_wr_err': 0, 'sum_wr_access': 186, 'file_family': 'daily-d0-offline', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.daily-d0-offline.cpio_odc', 'modification_time': 1257400918.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1257365852.0}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702075233_s103280_p2', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 1966080000L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7865, 'mc_device': '0,6,1,1', 'compression': 0, 'serial_num': '9310801322', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr205a', 'max_time_in_state': 1200, 'device': 'stkenmvr205a:/dev/rmt/tps0d0n', 'name': 'LTO4_105.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_105.stat', 'mover_address': ('131.225.13.201', 7865), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.201', 42203), 'data_ip': 'stkenmvr205a', 'logname': 'LTO4_105MV', 'hostip': '131.225.13.201', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702075233_s103280_p2', 'uid': 0, 'uname': 'root', 'mtime': 1257400876, 'rmajor': 0, 'machine': ('Linux', 'stkendca18a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 820618800L, 20L, 1, 1602, 2752, 512L, 1257400847, 1257400847, 1254398418), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 1966080000L, 'gname': 'root', 'gid': 0, 'fullname': '/diska/write-pool-1/data/0030000000000000061EB148', 'inode': 807317832L, 'minor': 20}, 'version': 'v3_8b  CVS $Revision: 1.922 $ <frozen>', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diska/write-pool-1/data/0030000000000000061EB148', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/.(access)(0030000000000000061EB148)', 'fc': {'uid': 0, 'external_label': 'VON266', 'gid': 0, 'location_cookie': '0000_000000000_0000187', 'complete_crc': 782989186L, 'bfid': 'CDMS125740103000000', 'sanity_cookie': (65536L, 4162356723L), 'size': 1966080000L}, 'infilepath': '/diska/write-pool-1/data/0030000000000000061EB148', 'at_the_top': 86, 'local_inode': 536955558L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.6', 54460), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257400874.730144, 'seek_time': 0.030066967010498047, 'transfer_time': 112.22035193443298, 't0': 1257400874, 'mount_time': 0.019438028335571289, 'lm_dequeued': 1257400918.22574, 'job_queued': 1257400876.62868, 'drive_call_time': 7.6963250637054443, 'in_queue': 41.597016096115112, 'drive_transfer_time': 19.191266059875488}, 'user_level': 'user2', 'unique_id': 'stkendca18a.fnal.gov-1257400876-21257-0'} Thread tape_thread
00:03:50 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  Done is sent Thread tape_thread
00:03:50 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  maybe_clean Thread tape_thread
00:03:50 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  returned from maybe_clean Thread tape_thread
00:03:50 stkendca18a.fnal.gov 021257 root I ENCP  Received final dialog for stkendca18a.fnal.gov-1257400876-21257-0.  elapsed=155.890sec
00:03:50 stkendca18a.fnal.gov 021257 root I ENCP  leaving transfer_file().
00:03:50 stkendca18a.fnal.gov 021257 root I ENCP  pf_status = ('ok', None)  skip_layer_cleanup = False
00:03:50 stkensrv0n.fnal.gov 028796 enstore I FILSRV  Updating  			update file set deleted = 'n', pnfs_path = '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702075233_s103280_p2', drive = 'stkenmvr205a:/dev/rmt/tps0d0n:9310801322', pnfs_id = '0030000000000000061EB148' where bfid = 'CDMS125740103000000';
00:03:50 stkendca18a.fnal.gov 021257 root I ENCP  MSG_TYPE=ENCP_XFER    write_to_hsm /diska/write-pool-1/data/0030000000000000061EB148 -> /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702075233_s103280_p2: 1966080000 bytes copied to VON266 at 16.7 MB/S (64 MB/S network) (97.7 MB/S drive) (17.4 MB/S disk) mover=LTO4_105.mover drive_id=ULTRIUM-TD4 drive_sn=9310801322 drive_vendor=IBM elapsed=156.2 {'storage_group': 'exp-db', 'encp_crc': 782989186L, 'disk_rate': 17.412698164168098, 'drive_rate': 97.70069333363017, 'driver': 'FTTDriver', 'transfer_rate': 16.701452297836042, 'mover_interface': 'stkenmvr205a', 'overall_rate': 16.645437295488279, 'media_changer': 'SL8500G1.media_changer', 'network_rate': 16.701452297836042, 'encp_ip': '131.225.13.6', 'unique_id': 'stkendca18a.fnal.gov-1257400876-21257-0'}
00:03:50 stkendca18a.fnal.gov 021257 root I ENCP  File /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702075233_s103280_p2 copied successfully.
00:03:50 stkendca18a.fnal.gov 021257 root I ENCP  Completed transferring 1966080000 bytes in 1 files in 156.214243889 sec.  Overall rate = 16.6 MB/sec.  Transfer rate = 16.7 MB/sec.  Network rate = 64 MB/sec.  Drive rate = 97.7 MB/sec.  Disk rate = 17.4 MB/sec.  Exit status = 0.
00:03:51 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:03:51 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  write_to_hsm next work on vol=VON266 mover=LTO4_105.mover requester:root
00:03:51 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  HAVE_BOUND:sending write_to_hsm /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702065710_s103261_p31 to mover LTO4_105.mover ('131.225.13.201', 7865) DEL_DISM None
00:03:51 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  MSG_TYPE=ADD_TO_LMQ  request added to work at movers queue for storage group : exp-db
00:03:51 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  HAVE_BOUND:sending {'vc': {'storage_group': 'exp-db', 'external_label': 'VON266', 'library': 'CD-LTO4G1', 'file_family': 'daily-d0-offline', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.daily-d0-offline.cpio_odc', 'file_family_width': 2}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702065710_s103261_p31', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 1966080000L, 'mover': 'LTO4_105.mover', 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702065710_s103261_p31', 'uid': 0, 'uname': 'root', 'type': 'cpio_odc', 'gid': 0, 'mtime': 1257400927, 'rmajor': 0, 'machine': ('Linux', 'stkendca18a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (16877, 820619016L, 20L, 1, 1602, 2752, 512L, 1257400914, 1257400914, 1254398418), 'mode_octal': '0100016', 'mode': 33188, 'gname': 'root', 'size_bytes': 1966080000L, 'fullname': '/diskb/write-pool-2/data/0030000000000000061EB048', 'inode': 807317576L, 'minor': 20}, 'version': 'v3_8b  CVS $Revision: 1.922 $ <frozen>', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskb/write-pool-2/data/0030000000000000061EB048', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/.(access)(0030000000000000061EB048)', 'fc': {'size': 1966080000L, 'external_label': 'VON266', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/diskb/write-pool-2/data/0030000000000000061EB048', 'at_the_top': 350, 'local_inode': 536871374L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.6', 54469), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257401031.3302369, 'encp_start_time': 1257400926.3717029, 'job_queued': 1257400927.9466181, 't0': 1257400926, 'in_queue': 103.38333296775818}, 'user_level': 'user2', 'unique_id': 'stkendca18a.fnal.gov-1257400927-22163-0'} to mover
00:03:51 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:03:51 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  WRITE_TO_HSM Thread MainThread
00:03:51 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  SENDING {'vc': {'storage_group': 'exp-db', 'external_label': 'VON266', 'library': 'CD-LTO4G1', 'file_family': 'daily-d0-offline', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.daily-d0-offline.cpio_odc', 'file_family_width': 2}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702065710_s103261_p31', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 1966080000L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7865, 'mc_device': '0,6,1,1', 'compression': 0, 'serial_num': '9310801322', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr205a', 'max_time_in_state': 1200, 'device': 'stkenmvr205a:/dev/rmt/tps0d0n', 'name': 'LTO4_105.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_105.stat', 'mover_address': ('131.225.13.201', 7865), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.201', 42206), 'data_ip': 'stkenmvr205a', 'logname': 'LTO4_105MV', 'hostip': '131.225.13.201', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702065710_s103261_p31', 'uid': 0, 'uname': 'root', 'mtime': 1257400927, 'rmajor': 0, 'machine': ('Linux', 'stkendca18a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 820619016L, 20L, 1, 1602, 2752, 512L, 1257400914, 1257400914, 1254398418), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 1966080000L, 'gname': 'root', 'gid': 0, 'fullname': '/diskb/write-pool-2/data/0030000000000000061EB048', 'inode': 807317576L, 'minor': 20}, 'version': 'v3_8b  CVS $Revision: 1.922 $ <frozen>', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskb/write-pool-2/data/0030000000000000061EB048', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/.(access)(0030000000000000061EB048)', 'fc': {'size': 1966080000L, 'external_label': 'VON266', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/diskb/write-pool-2/data/0030000000000000061EB048', 'at_the_top': 350, 'local_inode': 536871374L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.6', 54469), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257401031.3302369, 'encp_start_time': 1257400926.3717029, 'job_queued': 1257400927.9466181, 't0': 1257400926, 'in_queue': 103.38333296775818}, 'user_level': 'user2', 'unique_id': 'stkendca18a.fnal.gov-1257400927-22163-0'} Thread client_connect_thread
00:03:51 stkendca18a.fnal.gov 022163 root I ENCP  Control socket ('131.225.13.6', 54469) is connected to ('131.225.13.201', 42207) for stkendca18a.fnal.gov-1257400927-22163-0.  elapsed=105.000sec
00:03:51 stkendca18a.fnal.gov 022163 root I ENCP  Received callback ticket from mover LTO4_105.mover for transfer stkendca18a.fnal.gov-1257400927-22163-0.  elapsed=105.010sec
00:03:51 stkendca18a.fnal.gov 022163 root I ENCP  Atempting to connect data socket to mover at ('131.225.13.201', 42206).
00:03:51 stkendca18a.fnal.gov 022163 root I ENCP  Connecting data socket to mover (('131.225.13.201', 42206)) with interface 131.225.13.6.  elapsed=105.010sec
00:03:51 stkendca18a.fnal.gov 022163 root I ENCP  Data socket ('131.225.13.6', 54475) is connected to ('131.225.13.201', 42206).  elapsed=105.010sec
00:03:51 stkendca18a.fnal.gov 022163 root I ENCP  Opening local file.
00:03:51 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  tape position: target 188 current 188 0 Thread seek_thread
00:03:51 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  write_tape starting, bytes_to_write=1966080512 Thread tape_thread
00:03:51 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV  Write starting. Tape VON266 absolute location in blocks 2649429 Thread tape_thread
00:03:51 stkendca18a.fnal.gov 022163 root I ENCP  pf_status = ('ok', None)  skip_layer_cleanup = False
00:03:51 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:03:52 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:03:52 last message repeated 1 times
00:03:52 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV  skip_fm 1444 Thread seek_thread
00:03:52 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV  read_tape starting, bytes_to_read=547296 Thread tape_thread
00:03:53 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:03:54 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV  read bytes 547527/547527, blocks 5 header 231 Thread tape_thread
00:03:54 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV  drive stats after read. Tape VOP775 position 1445 block 5 block_size 0 bloc_loc 6267517 tot_blocks 762939000 BOT 0 read_err 0 write_err 0 bytes 547527 block_read_tot 1.31369805336 tape_rate 416782.987993 Thread tape_thread
00:03:54 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV  read_tape exiting, read 547527/547527 bytes Thread tape_thread
00:03:54 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV  read_tape timing:get_space 0.000341176986694 read 1.31084418297 crc_check 0.00246357917786 push 4.91142272949e-05 block_read 1.31369805336 idle 0.00010085105896 Thread tape_thread
00:03:54 stkensrv3n.fnal.gov 020327 root I ENCP  File /pnfs/cms/WAX/7/PCP04/Hit/eg03_jets_2g_pt50170/EVD0_Events.56914b76158411d998e90002b3d8e854.63719000.eg03_jets_2g_pt50170.eg_Hit245_2_g133 transfered for stkensrv3n.fnal.gov-1257400930-20327-0 in 0.00746917724609 seconds.
00:03:54 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV  write_client: wrote 547296/547296 bytes Thread net_thread
00:03:54 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV  write_client exits Thread net_thread
00:03:54 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV  transfer complete volume=VOP775 location=1445 Thread net_thread
00:03:54 last message repeated 1 times
00:03:54 stkensrv0n.fnal.gov 028762 enstore I VOLSRV  Updating  			update volume set sum_rd_access = '1448', last_access = '2009-11-05 00:03:54' where label = 'VOP775';
00:03:54 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV  Sending done to client: {'vc': {'comment': '<= VO5281 VO7669 VO9276 VO9363 VO9434 VO9484', 'declared': 1231186074.0, 'blocksize': 131072, 'storage_group': 'cms', 'library': 'CD-LTO4G1', 'eod_cookie': '0000_000000000_0001446', 'current_location': '0000_000000000_0001445', 'user_inhibit': ['none', 'none'], 'sum_rd_access': 1447, 'system_inhibit': ['none', 'full'], 'external_label': 'VOP775', 'wrapper': 'cpio_odc', 'remaining_bytes': 1525791744L, 'sum_mounts': 6, 'capacity_bytes': 858993459200L, 'media_type': 'LTO4', 'last_access': 1257400928.0, 'status': ('ok', None), 'si_time': [0.0, 1235726759.0], 'non_del_files': 1481, 'sum_wr_err': 0, 'sum_wr_access': 1445, 'file_family': 'cms7', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.cms7.cpio_odc', 'modification_time': 1238376412.0, 'write_protected': 'y', 'sum_rd_err': 0, 'first_access': 1235693914.0}, 'outfilepath': '/dev/null', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': 1, 'curpri': 1}, 'file_size': 547296L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'dismount_delay': 30, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7846, 'mc_device': '0,6,1,12', 'compression': 0, 'serial_num': '9311032672', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr146a', 'max_time_in_state': 1200, 'device': 'stkenmvr146a:/dev/rmt/tps0d0n', 'name': 'LTO4_46.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_46.stat', 'single_filemark': 1, 'mount_delay': 15, 'mover_address': ('131.225.13.174', 7846), 'check_first_written_file': 0, 'callback_addr': ('131.225.13.174', 44200), 'data_ip': 'stkenmvr146a', 'logname': 'LTO4_46MV', 'hostip': '131.225.13.174', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer', 'min_buffer': 67108864L}, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/cms/WAX/7/PCP04/Hit/eg03_jets_2g_pt50170/EVD0_Events.56914b76158411d998e90002b3d8e854.63719000.eg03_jets_2g_pt50170.eg_Hit245_2_g133', 'uid': 0, 'uname': 'root', 'rmajor': 0, 'machine': ('Linux', 'stkensrv3n.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 08:50:41 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'pstat': (33188, 34142048L, 20L, 1, 9811, 5063, 547296L, 1238376412, 1113792704, 1113792251), 'mode_octal': '0100744', 'mode': 33252, 'size_bytes': 547296L, 'gname': 'root', 'gid': 0, 'fullname': '/dev/null', 'inode': 34142048L, 'minor': 20}, 'version': 'v3_7d  CVS $Revision: 1.929 $ encp', 'encp_daq': None, 'client_crc': 1, 'infile': '/pnfs/cms/WAX/7/PCP04/Hit/eg03_jets_2g_pt50170/.(access)(00020000000000000008F760)', 'status': ('ok', None), 'volume': 'VOP775', 'outfile': '/dev/null', 'fc': {'status': ('ok', None), 'pnfsid': '00020000000000000008F760', 'pnfs_name0': '/pnfs/fs/usr/cms/WAX/7/PCP04/Hit/eg03_jets_2g_pt50170/EVD0_Events.56914b76158411d998e90002b3d8e854.63719000.eg03_jets_2g_pt50170.eg_Hit245_2_g133', 'deleted': 'no', 'external_label': 'VOP775', 'drive': 'stkenmvr148a:/dev/rmt/tps0d0n:9310060094', 'update': '2009-02-27 03:25:56.475019', 'gid': 0, 'location_cookie': '0000_000000000_0001445', 'complete_crc': 1644974717L, 'address': ('131.225.13.59', 7501), 'uid': 0, 'bfid': 'CDMS123572675300001', 'sanity_cookie': (65536L, 3819061325L), 'size': 547296L}, 'infilepath': '/pnfs/cms/WAX/7/PCP04/Hit/eg03_jets_2g_pt50170/EVD0_Events.56914b76158411d998e90002b3d8e854.63719000.eg03_jets_2g_pt50170.eg_Hit245_2_g133', 'at_the_top': 9, 'bfid': 'CDMS123572675300001', 'local_inode': 2469L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': 7, 'retry': 0}, 'callback_addr': ('131.225.13.51', 36450), 'work': 'read_from_hsm', 'override_ro_mount': 1, 'times': {'encp_start_time': 1257400929.1784661, 'seek_time': 101.71133303642273, 'transfer_time': 1.3779029846191406, 't0': 1257400929, 'mo(trunc. 4374) Thread net_thread
00:03:54 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV  Done is sent Thread net_thread
00:03:54 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV  maybe_clean Thread net_thread
00:03:54 stkensrv3n.fnal.gov 020327 root I ENCP  Received final dialog for stkensrv3n.fnal.gov-1257400930-20327-0.  elapsed=105.240sec
00:03:54 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV  returned from maybe_clean Thread net_thread
00:03:54 stkensrv3n.fnal.gov 020327 root I ENCP  leaving transfer_file().
00:03:54 stkensrv3n.fnal.gov 020327 root I ENCP  MSG_TYPE=ENCP_XFER    read_from_hsm /pnfs/cms/WAX/7/PCP04/Hit/eg03_jets_2g_pt50170/EVD0_Events.56914b76158411d998e90002b3d8e854.63719000.eg03_jets_2g_pt50170.eg_Hit245_2_g133 -> /dev/null: 547296 bytes copied from VOP775 at 8.18 MB/S (70.7 MB/S network) (0.397 MB/S drive) (70.7 MB/S disk) mover=LTO4_46.mover drive_id=ULTRIUM-TD4 drive_sn=9311032672 drive_vendor=IBM elapsed=105.25 {'storage_group': 'cms', 'encp_crc': 1644974717L, 'disk_rate': 70.704645826869879, 'drive_rate': 0.39728994447300714, 'driver': 'FTTDriver', 'transfer_rate': 8.1825806789214397, 'mover_interface': 'stkenmvr146a', 'overall_rate': 0.0050580383412459299, 'media_changer': 'SL8500G1.media_changer', 'network_rate': 8.1825806789214397, 'encp_ip': '131.225.13.51', 'unique_id': 'stkensrv3n.fnal.gov-1257400930-20327-0'}
00:03:54 stkensrv3n.fnal.gov 020327 root I ENCP  File /dev/null copied successfully.
00:03:54 stkensrv3n.fnal.gov 020327 root I ENCP  Completed transferring 547296 bytes in 1 files in 105.252379894 sec.  Overall rate = 0.00506 MB/sec.  Transfer rate = 8.18 MB/sec.  Network rate = 70.7 MB/sec.  Drive rate = 0.397 MB/sec.  Disk rate = 70.7 MB/sec.  Exit status = 0.
00:03:55 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:03:55 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:03:57 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:03:57 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:03:59 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_42.mover', 'VON743'), ('LTO4_105.mover', 'VON266')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:03:59 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:04:00 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:04:01 last message repeated 1 times
00:04:01 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:04:03 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:04:03 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:04:05 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:04:05 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:04:07 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:04:07 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:04:09 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:04:09 stkensrv4n.fnal.gov 007028 enstore I SL8500F1MC  mount VOO096 1,2,1,8 => 0,Mount: VOO096 mounted on   1, 2, 1, 8
00:04:09 stkensrv4n.fnal.gov 005843 enstore I SL8500F1MC  FINISHED mount VOO096 1,2,1,8 returned ('ok', 0, 'mount VOO096 1,2,1,8 => 0,Mount: VOO096 mounted on   1, 2, 1, 8', '', '')
00:04:09 last message repeated 1 times
00:04:09 stkensrv0n.fnal.gov 028762 enstore I VOLSRV  Updating  			update volume set sum_mounts = '1293', last_access = '2009-11-05 00:04:09' where label = 'VOO096';
00:04:09 stkenmvr217a.fnal.gov 007075 root I LTO4_117MV  MSG_TYPE=MC_LOAD_DONE  mounted VOO096 ULTRIUM-TD4 00:04:09 Thread media_thread
00:04:09 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:04:11 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:04:11 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:04:13 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:04:13 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:04:15 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:04:15 stkenmvr35a.fnal.gov 005964 root I DBT35MV  read bytes 4242341652/4242341652, blocks 32367 header 0 Thread tape_thread
00:04:15 stkenmvr35a.fnal.gov 005964 root I DBT35MV  write_client: wrote 4242341652/4242341652 bytes Thread net_thread
00:04:15 stkenmvr35a.fnal.gov 005964 root I DBT35MV  write_client waits for tape thread to exit Thread net_thread
00:04:15 stkenmvr35a.fnal.gov 005964 root I DBT35MV  drive stats after read. Tape VO6341 position 23 block 32367 block_size 0 bloc_loc 388040 tot_blocks 0 BOT 0 read_err 0 write_err 0 bytes 4242341652 block_read_tot 133.408964157 tape_rate 31799524.7081 Thread tape_thread
00:04:15 stkenmvr35a.fnal.gov 005964 root I DBT35MV  read_tape exiting, read 4242341652/4242341652 bytes Thread tape_thread
00:04:15 stkenmvr35a.fnal.gov 005964 root I DBT35MV  read_tape timing:get_space 0.506960868835 read 118.503316164 crc_check 13.9754412174 push 0.42324590683 block_read 133.408964157 idle 1.13599419594 Thread tape_thread
00:04:15 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:04:16 stkendca18a.fnal.gov 023073 root I ENCP  Version: v3_8b  CVS $Revision: 1.922 $ <frozen>  OS: Linux 2.6.9-89.0.9.ELsmp i686 Release:  Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0)  Group: root(0)  Euser: root(0)  Egroup: root(0)  Library: CD-LTO4G1  Storage Group: exp-db  File Family: daily-d0-offline  FF Wrapper: cpio_odc  FF Width: 2  Current working directory: stkendca18a.fnal.gov:/home/enstore/dcache-code/dcache-fermi-config  Command line: /fnal/ups/prd/encp/v3_8b/Linux-2-6-dcache/encp --verbose 4 --threaded --ecrc --bypass-filesystem-max-filesize-check --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702070562_s103266_p17 --put-cache 0030000000000000061EB2C0 /diska/write-pool-1/data/0030000000000000061EB2C0
00:04:16 stkendca18a.fnal.gov 023067 root I ENCP  Version: v3_8b  CVS $Revision: 1.922 $ <frozen>  OS: Linux 2.6.9-89.0.9.ELsmp i686 Release:  Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0)  Group: root(0)  Euser: root(0)  Egroup: root(0)  Library: CD-LTO4G1  Storage Group: exp-db  File Family: daily-d0-offline  FF Wrapper: cpio_odc  FF Width: 2  Current working directory: stkendca18a.fnal.gov:/home/enstore/dcache-code/dcache-fermi-config  Command line: /fnal/ups/prd/encp/v3_8b/Linux-2-6-dcache/encp --verbose 4 --threaded --ecrc --bypass-filesystem-max-filesize-check --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702075055_s103278_p2 --put-cache 0030000000000000061EB280 /diska/write-pool-1/data/0030000000000000061EB280
00:04:16 stkenmvr35a.fnal.gov 005964 root I DBT35MV  write_client exits Thread net_thread
00:04:16 stkenmvr35a.fnal.gov 005964 root I DBT35MV  transfer complete volume=VO6341 location=23 Thread net_thread
00:04:16 last message repeated 1 times
00:04:16 stkensrv0n.fnal.gov 028762 enstore I VOLSRV  Updating  			update volume set sum_rd_access = '96', last_access = '2009-11-05 00:04:16' where label = 'VO6341';
00:04:16 stkenmvr35a.fnal.gov 005964 root I DBT35MV  Sending done to client: {'vc': {'comment': '', 'declared': 1083263105.0, 'blocksize': 131072, 'storage_group': 'ktev', 'library': 'CD-9940B', 'eod_cookie': '0000_000000000_0000172', 'current_location': '0000_000000000_0000023', 'user_inhibit': ['none', 'none'], 'sum_rd_access': 95, 'system_inhibit': ['none', 'migrating'], 'external_label': 'VO6341', 'wrapper': 'cern', 'remaining_bytes': 6809693696L, 'sum_mounts': 54, 'capacity_bytes': 214748364800L, 'media_type': '9940B', 'last_access': 1257400917.0, 'status': ('ok', None), 'si_time': [0.0, 1257399311.0], 'non_del_files': 57, 'sum_wr_err': 0, 'sum_wr_access': 57, 'file_family': 'ktev_migration', 'address': ('131.225.13.59', 7502), 'volume_family': 'ktev.ktev_migration.cern', 'modification_time': 1193243360.0, 'write_protected': 'y', 'sum_rd_err': 0, 'first_access': 1088544231.0}, 'outfilepath': '/data/data2/Migration_Spool/VO6341:0000_000000000_0000023', 'encp': {'agetime': 30, 'adminpri': 6, 'delpri': 1, 'basepri': 1000007, 'delayed_dismount': 2, 'curpri': 1000007}, 'file_size': 4242341652L, 'mover': {'max_buffer': 1048576000L, 'check_written_file': 10, 'max_failures': 5, 'library': 'CD-9940B.library_manager', 'local_mover': 0, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7835, 'mc_device': '0,0,10,1', 'compression': 0, 'serial_num': '479001001549', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 28311552.0, 'vendor_id': 'STK', 'driver': 'FTTDriver', 'host': 'stkenmvr35a', 'max_time_in_state': 1200, 'device': 'stkenmvr35a:/dev/rmt/tps2d0n', 'name': '9940B35.mover', 'send_stats': 1, 'product_id': 'T9940B', 'statistics_path': '/tmp/enstore/enstore/DBT35MV.stat', 'mover_address': ('131.225.13.87', 7835), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.87', 33888), 'data_ip': 'stkenmvr35a', 'logname': 'DBT35MV', 'hostip': '131.225.13.87', 'do_cleaning': 'No', 'media_changer': 'stk.media_changer'}, 'lm': {'address': ('131.225.13.4', 7522)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC083/f0', 'uid': 0, 'uname': 'root', 'rmajor': 0, 'machine': ('Linux', 'stkendm7a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (33188, 525142144L, 20L, 1, 5406, 2539, 1L, 1177383248, 1088548411, 1088546827), 'mode_octal': '0100744', 'mode': 33252, 'size_bytes': 4242341652L, 'gname': 'root', 'gid': 0, 'fullname': '/data/data2/Migration_Spool/VO6341:0000_000000000_0000023', 'inode': 525142144L, 'minor': 20}, 'version': 'v3_8  CVS $Revision: 1.963 $ encp.pyc', 'encp_daq': None, 'client_crc': 1, 'infile': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC083/.(access)(001F000000000000014D0880)', 'status': ('ok', None), 'volume': 'VO6341', 'outfile': '/data/data2/Migration_Spool/VO6341:0000_000000000_0000023', 'fc': {'status': ('ok', None), 'pnfsid': '001F000000000000014D0880', 'pnfs_name0': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC083/f0', 'deleted': 'no', 'external_label': 'VO6341', 'drive': 'stkenmvr10a:/dev/rmt/tps0d0n:4790000020', 'update': None, 'gid': 0, 'location_cookie': '0000_000000000_0000023', 'complete_crc': 701935742L, 'address': ('131.225.13.59', 7501), 'uid': 0, 'bfid': 'CDMS108854841100000', 'sanity_cookie': (65536L, 2360529580L), 'size': 4242341652L}, 'infilepath': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC083/f0', 'at_the_top': 6, 'bfid': 'CDMS108854841100000', 'local_inode': 4472841L, 'resend': {'max_retry': 3, 'resubmits': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.19', 50809), 'work': 'read_from_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257400552.5200961, 'seek_time': 0.0061202049255371094, 'transfer_time': 137.9399528503418, 't0': 1257400552, 'mount_time': 0.020724773406982422, 'lm_dequeued': 1257400918.5814681, 'job_queued': 1257400552.7230361, 'drive_call_time': 117.76843166351318, 'in_queue': 365.85801196098328, 'drive_transfer_time': 133.89994955062866}, 'user_level(trunc. 4145) Thread net_thread
00:04:16 stkenmvr35a.fnal.gov 005964 root I DBT35MV  Done is sent Thread net_thread
00:04:16 stkenmvr35a.fnal.gov 005964 root I DBT35MV  maybe_clean Thread net_thread
00:04:16 stkenmvr35a.fnal.gov 005964 root I DBT35MV  returned from maybe_clean Thread net_thread
00:04:16 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  <storage groups {'ktev': [('9940B10.mover', 'VO6332'), ('9940B35.mover', 'VO6341')]} volume_families {'ktev.ktev_migration.cern': [('9940B10.mover', 'VO6332'), ('9940B35.mover', 'VO6341')]} >
00:04:16 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  read_from_hsm next work on vol=VO6341 mover=9940B35.mover requester:root
00:04:16 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  HAVE_BOUND:sending read_from_hsm /pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC084/f2 to mover 9940B35.mover ('131.225.13.87', 7835) DEL_DISM 2
00:04:16 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  MSG_TYPE=ADD_TO_LMQ  request added to work at movers queue for storage group : ktev
00:04:16 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  HAVE_BOUND:sending {'vc': {'comment': '', 'declared': 1083263105.0, 'blocksize': 131072, 'storage_group': 'ktev', 'library': 'CD-9940B', 'eod_cookie': '0000_000000000_0000172', 'current_location': '0000_000000000_0000026', 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'migrating'], 'external_label': 'VO6341', 'wrapper': 'cern', 'remaining_bytes': 6809693696L, 'sum_mounts': 54, 'sum_rd_access': 94, 'media_type': '9940B', 'last_access': 1257400916.0, 'status': ('ok', None), 'si_time': [0.0, 1257399311.0], 'capacity_bytes': 214748364800L, 'non_del_files': 57, 'sum_wr_err': 0, 'sum_wr_access': 57, 'file_family': 'ktev_migration', 'address': ('131.225.13.59', 7502), 'volume_family': 'ktev.ktev_migration.cern', 'modification_time': 1193243360.0, 'write_protected': 'y', 'sum_rd_err': 0, 'first_access': 1088544231.0}, 'outfilepath': '/data/data2/Migration_Spool/VO6341:0000_000000000_0000026', 'encp': {'agetime': 30, 'adminpri': 6, 'delpri': 1, 'basepri': 1000007, 'delayed_dismount': 2, 'curpri': 1000007}, 'file_size': 21756L, 'mover': '9940B35.mover', 'lm': {'address': ('131.225.13.4', 7522)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC084/f2', 'uid': 0, 'uname': 'root', 'gid': 0, 'rmajor': 0, 'machine': ('Linux', 'stkendm7a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (33188, 525142592L, 20L, 1, 5406, 2539, 21756L, 1177388762, 1088549272, 1088549129), 'mode_octal': '0100744', 'mode': 33252, 'gname': 'root', 'size_bytes': 21756L, 'fullname': '/data/data2/Migration_Spool/VO6341:0000_000000000_0000026', 'inode': 525142592L, 'minor': 20}, 'version': 'v3_8  CVS $Revision: 1.963 $ encp.pyc', 'encp_daq': None, 'client_crc': 1, 'infile': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC084/.(access)(001F000000000000014D0A40)', 'status': ('ok', None), 'volume': 'VO6341', 'outfile': '/data/data2/Migration_Spool/VO6341:0000_000000000_0000026', 'fc': {'status': ('ok', None), 'pnfsid': '001F000000000000014D0A40', 'pnfs_name0': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC084/f2', 'deleted': 'no', 'external_label': 'VO6341', 'drive': 'stkenmvr26a:/dev/rmt/tps0d0n:4790000200', 'update': None, 'gid': 0, 'location_cookie': '0000_000000000_0000026', 'complete_crc': 3155301115L, 'address': ('131.225.13.59', 7501), 'uid': 0, 'bfid': 'CDMS108854927200000', 'sanity_cookie': (21756L, 3155301115L), 'size': 21756L}, 'infilepath': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC084/f2', 'at_the_top': 418, 'bfid': 'CDMS108854927200000', 'local_inode': 4472842L, 'resend': {'max_retry': 3, 'resubmits': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.19', 50825), 'work': 'read_from_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257401056.683321, 'encp_start_time': 1257400916.734205, 'job_queued': 1257400916.90485, 't0': 1257400916, 'in_queue': 139.77801489830017}, 'user_level': 'admin', 'unique_id': 'stkendm7a.fnal.gov-1257400916-29514-8'} to mover
00:04:16 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  <storage groups {'ktev': [('9940B10.mover', 'VO6332'), ('9940B35.mover', 'VO6341')]} volume_families {'ktev.ktev_migration.cern': [('9940B10.mover', 'VO6332'), ('9940B35.mover', 'VO6341')]} >
00:04:16 stkendca18a.fnal.gov 023073 root I ENCP  Creating zero length output files.
00:04:16 stkendca18a.fnal.gov 023067 root I ENCP  Creating zero length output files.
00:04:16 stkendca18a.fnal.gov 023073 root I ENCP  Sending /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702070562_s103266_p17 write request to LM: unique_id: stkendca18a.fnal.gov-1257401056-23073-0 inputfile: /diska/write-pool-1/data/0030000000000000061EB2C0 outputfile: /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/.(access)(0030000000000000061EB2C0)
00:04:16 stkenmvr35a.fnal.gov 005964 root I DBT35MV  READ FROM HSM Thread MainThread
00:04:16 stkenmvr35a.fnal.gov 005964 root I DBT35MV  SENDING {'vc': {'comment': '', 'declared': 1083263105.0, 'blocksize': 131072, 'storage_group': 'ktev', 'library': 'CD-9940B', 'eod_cookie': '0000_000000000_0000172', 'current_location': '0000_000000000_0000026', 'user_inhibit': ['none', 'none'], 'sum_rd_access': 94, 'system_inhibit': ['none', 'migrating'], 'external_label': 'VO6341', 'wrapper': 'cern', 'remaining_bytes': 6809693696L, 'sum_mounts': 54, 'capacity_bytes': 214748364800L, 'media_type': '9940B', 'last_access': 1257400916.0, 'status': ('ok', None), 'si_time': [0.0, 1257399311.0], 'non_del_files': 57, 'sum_wr_err': 0, 'sum_wr_access': 57, 'file_family': 'ktev_migration', 'address': ('131.225.13.59', 7502), 'volume_family': 'ktev.ktev_migration.cern', 'modification_time': 1193243360.0, 'write_protected': 'y', 'sum_rd_err': 0, 'first_access': 1088544231.0}, 'outfilepath': '/data/data2/Migration_Spool/VO6341:0000_000000000_0000026', 'encp': {'agetime': 30, 'adminpri': 6, 'delpri': 1, 'basepri': 1000007, 'delayed_dismount': 2, 'curpri': 1000007}, 'file_size': 21756L, 'mover': {'max_buffer': 1048576000L, 'check_written_file': 10, 'max_failures': 5, 'library': 'CD-9940B.library_manager', 'local_mover': 0, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7835, 'mc_device': '0,0,10,1', 'compression': 0, 'serial_num': '479001001549', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 28311552.0, 'vendor_id': 'STK', 'driver': 'FTTDriver', 'host': 'stkenmvr35a', 'max_time_in_state': 1200, 'device': 'stkenmvr35a:/dev/rmt/tps2d0n', 'name': '9940B35.mover', 'send_stats': 1, 'product_id': 'T9940B', 'statistics_path': '/tmp/enstore/enstore/DBT35MV.stat', 'mover_address': ('131.225.13.87', 7835), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.87', 33896), 'data_ip': 'stkenmvr35a', 'logname': 'DBT35MV', 'hostip': '131.225.13.87', 'do_cleaning': 'No', 'media_changer': 'stk.media_changer'}, 'lm': {'address': ('131.225.13.4', 7522)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC084/f2', 'uid': 0, 'uname': 'root', 'rmajor': 0, 'machine': ('Linux', 'stkendm7a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (33188, 525142592L, 20L, 1, 5406, 2539, 21756L, 1177388762, 1088549272, 1088549129), 'mode_octal': '0100744', 'mode': 33252, 'size_bytes': 21756L, 'gname': 'root', 'gid': 0, 'fullname': '/data/data2/Migration_Spool/VO6341:0000_000000000_0000026', 'inode': 525142592L, 'minor': 20}, 'version': 'v3_8  CVS $Revision: 1.963 $ encp.pyc', 'encp_daq': None, 'client_crc': 1, 'infile': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC084/.(access)(001F000000000000014D0A40)', 'status': ('ok', None), 'volume': 'VO6341', 'outfile': '/data/data2/Migration_Spool/VO6341:0000_000000000_0000026', 'fc': {'status': ('ok', None), 'pnfsid': '001F000000000000014D0A40', 'pnfs_name0': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC084/f2', 'deleted': 'no', 'external_label': 'VO6341', 'drive': 'stkenmvr26a:/dev/rmt/tps0d0n:4790000200', 'update': None, 'gid': 0, 'location_cookie': '0000_000000000_0000026', 'complete_crc': 3155301115L, 'address': ('131.225.13.59', 7501), 'uid': 0, 'bfid': 'CDMS108854927200000', 'sanity_cookie': (21756L, 3155301115L), 'size': 21756L}, 'infilepath': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC084/f2', 'at_the_top': 418, 'bfid': 'CDMS108854927200000', 'local_inode': 4472842L, 'resend': {'max_retry': 3, 'resubmits': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.19', 50825), 'work': 'read_from_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257401056.683321, 'encp_start_time': 1257400916.734205, 'job_queued': 1257400916.90485, 't0': 1257400916, 'in_queue': 139.77801489830017}, 'user_level': 'admin', 'unique_id': 'stkendm7a.fnal.gov-1257400916-29514-8'} Thread client_connect_thread
00:04:16 stkendm7a.fnal.gov 029514 root I ENCP  Control socket ('131.225.13.19', 50825) is connected to ('131.225.13.87', 33897) for stkendm7a.fnal.gov-1257400916-29514-8.  elapsed=1744.480sec Thread READ_2
00:04:16 stkendm7a.fnal.gov 029514 root I ENCP  Received callback ticket from mover 9940B35.mover for transfer stkendm7a.fnal.gov-1257400916-29514-8.  elapsed=1744.480sec Thread READ_2
00:04:16 stkendm7a.fnal.gov 029514 root I ENCP  Attempting to connect data socket to mover at ('131.225.13.87', 33896). Thread READ_2
00:04:16 stkendm7a.fnal.gov 029514 root I ENCP  Connecting data socket to mover (('131.225.13.87', 33896)) with interface 131.225.13.19.  elapsed=1744.480sec Thread READ_2
00:04:16 stkendm7a.fnal.gov 029514 root I ENCP  Data socket ('131.225.13.19', 50831) is connected to ('131.225.13.87', 33896).  elapsed=1744.490sec Thread READ_2
00:04:16 stkendca18a.fnal.gov 023067 root I ENCP  Sending /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702075055_s103278_p2 write request to LM: unique_id: stkendca18a.fnal.gov-1257401056-23067-0 inputfile: /diska/write-pool-1/data/0030000000000000061EB280 outputfile: /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/.(access)(0030000000000000061EB280)
00:04:16 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  MSG_TYPE=ADD_TO_LMQ  request added to pending queue for storage group : exp-db
00:04:16 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  write Q'd /diska/write-pool-1/data/0030000000000000061EB2C0 (stkendca18a.fnal.gov-1257401056-23073-0) -> /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel02/D0OFPRD1_t702070562_s103266_p17 : library=CD-LTO4G1 family=daily-d0-offline requester:root volume_family:exp-db.daily-d0-offline.cpio_odc
00:04:16 stkendm7a.fnal.gov 029514 root I ENCP  Opening local file. Thread READ_2
00:04:16 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  MSG_TYPE=ADD_TO_LMQ  request added to pending queue for storage group : exp-db
00:04:16 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  write Q'd /diska/write-pool-1/data/0030000000000000061EB280 (stkendca18a.fnal.gov-1257401056-23067-0) -> /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/04/channel01/D0OFPRD1_t702075055_s103278_p2 : library=CD-LTO4G1 family=daily-d0-offline requester:root volume_family:exp-db.daily-d0-offline.cpio_odc
00:04:16 stkenmvr35a.fnal.gov 005964 root I DBT35MV  tape position: target 26 current 23 32367 Thread seek_thread
00:04:16 stkenmvr35a.fnal.gov 005964 root I DBT35MV  skip_fm 3 Thread seek_thread
00:04:16 stkendm7a.fnal.gov 029514 root I ENCP  File /pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC083/f0 transfered for stkendm7a.fnal.gov-1257400552-29514-7 in 138.089462996 seconds. Thread READ_1
00:04:16 stkenmvr35a.fnal.gov 005964 root I DBT35MV  read_tape starting, bytes_to_read=21756 Thread tape_thread
00:04:16 stkendm7a.fnal.gov 029514 root I ENCP  Received final dialog for stkendm7a.fnal.gov-1257400552-29514-7.  elapsed=1744.510sec Thread READ_1
00:04:16 stkenmvr35a.fnal.gov 005964 root I DBT35MV  read bytes 21756/21756, blocks 1 header 0 Thread tape_thread
00:04:16 stkendca18a.fnal.gov 023073 root I ENCP  Listening for control socket at: ('131.225.13.6', 54479)
00:04:16 stkendca18a.fnal.gov 023073 root I ENCP  Waiting for mover to connect control socket at address: ('131.225.13.6', 54479)  elapsed=1.740sec
00:04:16 stkenmvr35a.fnal.gov 005964 root I DBT35MV  write_client: wrote 21756/21756 bytes Thread net_thread
00:04:16 stkenmvr35a.fnal.gov 005964 root I DBT35MV  write_client waits for tape thread to exit Thread net_thread
00:04:16 stkendm7a.fnal.gov 029514 root I ENCP  leaving transfer_file(). Thread READ_1
00:04:16 stkenmvr35a.fnal.gov 005964 root I DBT35MV  drive stats after read. Tape VO6341 position 26 block 1 block_size 0 bloc_loc 388046 tot_blocks 0 BOT 0 read_err 0 write_err 0 bytes 21756 block_read_tot 0.00270199775696 tape_rate 8051820.15565 Thread tape_thread
00:04:16 stkendm7a.fnal.gov 029514 root I ENCP  File /pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC084/f2 transfered for stkendm7a.fnal.gov-1257400916-29514-8 in 0.00397706031799 seconds. Thread READ_2
00:04:16 stkenmvr35a.fnal.gov 005964 root I DBT35MV  read_tape exiting, read 21756/21756 bytes Thread tape_thread
00:04:16 stkenmvr35a.fnal.gov 005964 root I DBT35MV  read_tape timing:get_space 0.000117063522339 read 0.00233888626099 crc_check 0.000227928161621 push 1.81198120117e-05 block_read 0.00270199775696 idle 3.69548797607e-05 Thread tape_thread
00:04:16 stkendca18a.fnal.gov 023067 root I ENCP  Listening for control socket at: ('131.225.13.6', 54480)
00:04:16 stkendca18a.fnal.gov 023067 root I ENCP  Waiting for mover to connect control socket at address: ('131.225.13.6', 54480)  elapsed=1.750sec
00:04:17 stkendm7a.fnal.gov 029514 root I ENCP  MSG_TYPE=ENCP_XFER    read_from_hsm /pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC083/f0 -> /data/data2/Migration_Spool/VO6341:0000_000000000_0000023: 4242341652 bytes copied from VO6341 at 29.3 MB/S (39.2 MB/S network) (30.2 MB/S drive) (63.6 MB/S disk) mover=9940B35.mover drive_id=T9940B drive_sn=479001001549 drive_vendor=STK elapsed=504.36 {'storage_group': 'ktev', 'encp_crc': 701935742L, 'disk_rate': 63.586641161158475, 'drive_rate': 30.215188941524634, 'driver': 'FTTDriver', 'transfer_rate': 29.296140341325586, 'mover_interface': 'stkenmvr35a', 'overall_rate': 29.266314281111622, 'media_changer': 'stk.media_changer', 'network_rate': 29.296140341325586, 'encp_ip': '131.225.13.19', 'unique_id': 'stkendm7a.fnal.gov-1257400552-29514-7'} Thread READ_1
00:04:17 stkendm7a.fnal.gov 029514 root I ENCP  File /data/data2/Migration_Spool/VO6341:0000_000000000_0000023 copied successfully. Thread READ_1
00:04:17 stkendm7a.fnal.gov 029514 root I ENCP  Completed transferring 4242341652 bytes in 1 files in 504.369710922 sec.  Overall rate = 29.3 MB/sec.  Transfer rate = 29.3 MB/sec.  Network rate = 39.2 MB/sec.  Drive rate = 30.2 MB/sec.  Disk rate = 63.6 MB/sec.  Exit status = 0. Thread READ_1
00:04:17 stkendm7a.fnal.gov 029514 root I ENCP  Version: v3_8  CVS $Revision: 1.963 $ encp.pyc  OS: Linux 2.6.9-89.0.9.ELsmp i686 Release:  Scientific Linux Fermi LTS release 4.4 (Wilson), User: root(0)  Group: root(0)  Euser: root(0)  Egroup: root(0)  Current working directory: stkendm7a.fnal.gov:/opt/enstore/src  Command line: encp --delayed-dismount 2 --ignore-fair-share --bypass-filesystem-max-filesize-check --priority 0 --threaded /pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC083/f1 /data/data2/Migration_Spool/VO6341:0000_000000000_0000032 Thread READ_1
00:04:17 stkendm7a.fnal.gov 029514 root I ENCP  Creating zero length output files. Thread READ_1
00:04:17 stkendm7a.fnal.gov 029514 root I ENCP  Sending /pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC083/f1 read request to LM: unique_id: stkendm7a.fnal.gov-1257401057-29514-10 inputfile: /pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC083/.(access)(001F000000000000014D0928) outputfile: /data/data2/Migration_Spool/VO6341:0000_000000000_0000032 Thread READ_1
00:04:17 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  MSG_TYPE=ADD_TO_LMQ  request added to pending queue for storage group : ktev
00:04:17 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  read Q'd /pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC083/f1 (stkendm7a.fnal.gov-1257401057-29514-10) -> /data/data2/Migration_Spool/VO6341:0000_000000000_0000032 : library=CD-9940B family=ktev.ktev_migration.cern requester:root
00:04:17 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:04:17 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  <storage groups {'ktev': [('9940B10.mover', 'VO6332'), ('9940B35.mover', 'VO6341')]} volume_families {'ktev.ktev_migration.cern': [('9940B10.mover', 'VO6332'), ('9940B35.mover', 'VO6341')]} >
00:04:17 stkenmvr35a.fnal.gov 005964 root I DBT35MV  write_client exits Thread net_thread
00:04:17 stkenmvr35a.fnal.gov 005964 root I DBT35MV  transfer complete volume=VO6341 location=26 Thread net_thread
00:04:17 last message repeated 1 times
00:04:17 stkensrv0n.fnal.gov 028762 enstore I VOLSRV  Updating  			update volume set sum_rd_access = '97', last_access = '2009-11-05 00:04:17' where label = 'VO6341';
00:04:17 stkenmvr35a.fnal.gov 005964 root I DBT35MV  Sending done to client: {'vc': {'comment': '', 'declared': 1083263105.0, 'blocksize': 131072, 'storage_group': 'ktev', 'library': 'CD-9940B', 'eod_cookie': '0000_000000000_0000172', 'current_location': '0000_000000000_0000026', 'user_inhibit': ['none', 'none'], 'sum_rd_access': 96, 'system_inhibit': ['none', 'migrating'], 'external_label': 'VO6341', 'wrapper': 'cern', 'remaining_bytes': 6809693696L, 'sum_mounts': 54, 'capacity_bytes': 214748364800L, 'media_type': '9940B', 'last_access': 1257401056.0, 'status': ('ok', None), 'si_time': [0.0, 1257399311.0], 'non_del_files': 57, 'sum_wr_err': 0, 'sum_wr_access': 57, 'file_family': 'ktev_migration', 'address': ('131.225.13.59', 7502), 'volume_family': 'ktev.ktev_migration.cern', 'modification_time': 1193243360.0, 'write_protected': 'y', 'sum_rd_err': 0, 'first_access': 1088544231.0}, 'outfilepath': '/data/data2/Migration_Spool/VO6341:0000_000000000_0000026', 'encp': {'agetime': 30, 'adminpri': 6, 'delpri': 1, 'basepri': 1000007, 'delayed_dismount': 2, 'curpri': 1000007}, 'file_size': 21756L, 'mover': {'max_buffer': 1048576000L, 'check_written_file': 10, 'max_failures': 5, 'library': 'CD-9940B.library_manager', 'local_mover': 0, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7835, 'mc_device': '0,0,10,1', 'compression': 0, 'serial_num': '479001001549', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 28311552.0, 'vendor_id': 'STK', 'driver': 'FTTDriver', 'host': 'stkenmvr35a', 'max_time_in_state': 1200, 'device': 'stkenmvr35a:/dev/rmt/tps2d0n', 'name': '9940B35.mover', 'send_stats': 1, 'product_id': 'T9940B', 'statistics_path': '/tmp/enstore/enstore/DBT35MV.stat', 'mover_address': ('131.225.13.87', 7835), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.87', 33896), 'data_ip': 'stkenmvr35a', 'logname': 'DBT35MV', 'hostip': '131.225.13.87', 'do_cleaning': 'No', 'media_changer': 'stk.media_changer'}, 'lm': {'address': ('131.225.13.4', 7522)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC084/f2', 'uid': 0, 'uname': 'root', 'rmajor': 0, 'machine': ('Linux', 'stkendm7a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (33188, 525142592L, 20L, 1, 5406, 2539, 21756L, 1177388762, 1088549272, 1088549129), 'mode_octal': '0100744', 'mode': 33252, 'size_bytes': 21756L, 'gname': 'root', 'gid': 0, 'fullname': '/data/data2/Migration_Spool/VO6341:0000_000000000_0000026', 'inode': 525142592L, 'minor': 20}, 'version': 'v3_8  CVS $Revision: 1.963 $ encp.pyc', 'encp_daq': None, 'client_crc': 1, 'infile': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC084/.(access)(001F000000000000014D0A40)', 'status': ('ok', None), 'volume': 'VO6341', 'outfile': '/data/data2/Migration_Spool/VO6341:0000_000000000_0000026', 'fc': {'status': ('ok', None), 'pnfsid': '001F000000000000014D0A40', 'pnfs_name0': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC084/f2', 'deleted': 'no', 'external_label': 'VO6341', 'drive': 'stkenmvr26a:/dev/rmt/tps0d0n:4790000200', 'update': None, 'gid': 0, 'location_cookie': '0000_000000000_0000026', 'complete_crc': 3155301115L, 'address': ('131.225.13.59', 7501), 'uid': 0, 'bfid': 'CDMS108854927200000', 'sanity_cookie': (21756L, 3155301115L), 'size': 21756L}, 'infilepath': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC084/f2', 'at_the_top': 418, 'bfid': 'CDMS108854927200000', 'local_inode': 4472842L, 'resend': {'max_retry': 3, 'resubmits': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.19', 50825), 'work': 'read_from_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257400916.734205, 'seek_time': 0.0062639713287353516, 'transfer_time': 1.0113799571990967, 't0': 1257400916, 'mount_time': 0.019250869750976562, 'lm_dequeued': 1257401056.683321, 'job_queued': 1257400916.90485, 'drive_call_time': 0.0023119449615478516, 'in_queue': 139.77801489830017, 'drive_transfer_time': 0.0027189254760742188}, 'user_level': 'a(trunc. 4140) Thread net_thread
00:04:17 stkenmvr35a.fnal.gov 005964 root I DBT35MV  Done is sent Thread net_thread
00:04:17 stkenmvr35a.fnal.gov 005964 root I DBT35MV  maybe_clean Thread net_thread
00:04:17 stkendm7a.fnal.gov 029514 root I ENCP  Received final dialog for stkendm7a.fnal.gov-1257400916-29514-8.  elapsed=1745.540sec Thread READ_2
00:04:17 stkendm7a.fnal.gov 029514 root I ENCP  leaving transfer_file(). Thread READ_2
00:04:17 stkenmvr35a.fnal.gov 005964 root I DBT35MV  returned from maybe_clean Thread net_thread
00:04:17 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:04:17 stkendm7a.fnal.gov 029514 root I ENCP  MSG_TYPE=ENCP_XFER    read_from_hsm /pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC084/f2 -> /data/data2/Migration_Spool/VO6341:0000_000000000_0000026: 21756 bytes copied from VO6341 at 0.0202 MB/S (163 MB/S network) (7.63 MB/S drive) (6.14 MB/S disk) mover=9940B35.mover drive_id=T9940B drive_sn=479001001549 drive_vendor=STK elapsed=141.13 {'storage_group': 'ktev', 'encp_crc': 3155301115L, 'disk_rate': 6.1403193645507859, 'drive_rate': 7.6310066643283054, 'driver': 'FTTDriver', 'transfer_rate': 0.020175674707585561, 'mover_interface': 'stkenmvr35a', 'overall_rate': 0.018405110387517416, 'media_changer': 'stk.media_changer', 'network_rate': 0.020175674707585561, 'encp_ip': '131.225.13.19', 'unique_id': 'stkendm7a.fnal.gov-1257400916-29514-8'} Thread READ_2
00:04:17 stkendm7a.fnal.gov 029514 root I ENCP  File /data/data2/Migration_Spool/VO6341:0000_000000000_0000026 copied successfully. Thread READ_2
00:04:17 stkendm7a.fnal.gov 029514 root I ENCP  Completed transferring 21756 bytes in 1 files in 141.143810987 sec.  Overall rate = 0.0184 MB/sec.  Transfer rate = 0.0202 MB/sec.  Network rate = 163 MB/sec.  Drive rate = 7.63 MB/sec.  Disk rate = 6.14 MB/sec.  Exit status = 0. Thread READ_2
00:04:18 stkendm7a.fnal.gov 029514 root I ENCP  Version: v3_8  CVS $Revision: 1.963 $ encp.pyc  OS: Linux 2.6.9-89.0.9.ELsmp i686 Release:  Scientific Linux Fermi LTS release 4.4 (Wilson), User: root(0)  Group: root(0)  Euser: root(0)  Egroup: root(0)  Current working directory: stkendm7a.fnal.gov:/opt/enstore/src  Command line: encp --delayed-dismount 2 --ignore-fair-share --bypass-filesystem-max-filesize-check --priority 0 --threaded /pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKS3/QKS369/f0 /data/data2/Migration_Spool/VO6341:0000_000000000_0000035 Thread READ_2
00:04:18 stkendm7a.fnal.gov 029514 root I ENCP  Creating zero length output files. Thread READ_2
00:04:18 stkendm7a.fnal.gov 029514 root I ENCP  Sending /pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKS3/QKS369/f0 read request to LM: unique_id: stkendm7a.fnal.gov-1257401058-29514-11 inputfile: /pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKS3/QKS369/.(access)(001F000000000000014D0C38) outputfile: /data/data2/Migration_Spool/VO6341:0000_000000000_0000035 Thread READ_2
00:04:18 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  MSG_TYPE=ADD_TO_LMQ  request added to pending queue for storage group : ktev
00:04:18 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  read Q'd /pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKS3/QKS369/f0 (stkendm7a.fnal.gov-1257401058-29514-11) -> /data/data2/Migration_Spool/VO6341:0000_000000000_0000035 : library=CD-9940B family=ktev.ktev_migration.cern requester:root
00:04:18 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  <storage groups {'ktev': [('9940B10.mover', 'VO6332'), ('9940B35.mover', 'VO6341')]} volume_families {'ktev.ktev_migration.cern': [('9940B10.mover', 'VO6332'), ('9940B35.mover', 'VO6341')]} >
00:04:18 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  read_from_hsm next work on vol=VO6341 mover=9940B35.mover requester:root
00:04:18 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  HAVE_BOUND:sending read_from_hsm /pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC084/f0 to mover 9940B35.mover ('131.225.13.87', 7835) DEL_DISM 2
00:04:18 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  MSG_TYPE=ADD_TO_LMQ  request added to work at movers queue for storage group : ktev
00:04:18 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  HAVE_BOUND:sending {'vc': {'comment': '', 'declared': 1083263105.0, 'blocksize': 131072, 'storage_group': 'ktev', 'library': 'CD-9940B', 'eod_cookie': '0000_000000000_0000172', 'current_location': '0000_000000000_0000029', 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'migrating'], 'external_label': 'VO6341', 'wrapper': 'cern', 'remaining_bytes': 6809693696L, 'sum_mounts': 54, 'sum_rd_access': 95, 'media_type': '9940B', 'last_access': 1257400917.0, 'status': ('ok', None), 'si_time': [0.0, 1257399311.0], 'capacity_bytes': 214748364800L, 'non_del_files': 57, 'sum_wr_err': 0, 'sum_wr_access': 57, 'file_family': 'ktev_migration', 'address': ('131.225.13.59', 7502), 'volume_family': 'ktev.ktev_migration.cern', 'modification_time': 1193243360.0, 'write_protected': 'y', 'sum_rd_err': 0, 'first_access': 1088544231.0}, 'outfilepath': '/data/data2/Migration_Spool/VO6341:0000_000000000_0000029', 'encp': {'agetime': 30, 'adminpri': 6, 'delpri': 1, 'basepri': 1000007, 'delayed_dismount': 2, 'curpri': 1000007}, 'file_size': 4235003240L, 'mover': '9940B35.mover', 'lm': {'address': ('131.225.13.4', 7522)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC084/f0', 'uid': 0, 'uname': 'root', 'gid': 0, 'rmajor': 0, 'machine': ('Linux', 'stkendm7a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (33188, 525142400L, 20L, 1, 5406, 2539, 1L, 1177385614, 1088549484, 1088547614), 'mode_octal': '0100744', 'mode': 33252, 'gname': 'root', 'size_bytes': 4235003240L, 'fullname': '/data/data2/Migration_Spool/VO6341:0000_000000000_0000029', 'inode': 525142400L, 'minor': 20}, 'version': 'v3_8  CVS $Revision: 1.963 $ encp.pyc', 'encp_daq': None, 'client_crc': 1, 'infile': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC084/.(access)(001F000000000000014D0980)', 'status': ('ok', None), 'volume': 'VO6341', 'outfile': '/data/data2/Migration_Spool/VO6341:0000_000000000_0000029', 'fc': {'status': ('ok', None), 'pnfsid': '001F000000000000014D0980', 'pnfs_name0': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC084/f0', 'deleted': 'no', 'external_label': 'VO6341', 'drive': 'stkenmvr26a:/dev/rmt/tps0d0n:4790000200', 'update': None, 'gid': 0, 'location_cookie': '0000_000000000_0000029', 'complete_crc': 1714586310L, 'address': ('131.225.13.59', 7501), 'uid': 0, 'bfid': 'CDMS108854948400000', 'sanity_cookie': (65536L, 3919951947L), 'size': 4235003240L}, 'infilepath': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC084/f0', 'at_the_top': 6, 'bfid': 'CDMS108854948400000', 'local_inode': 4472843L, 'resend': {'max_retry': 3, 'resubmits': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.19', 50827), 'work': 'read_from_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257401058.684545, 'encp_start_time': 1257400917.93977, 'job_queued': 1257400918.0959611, 't0': 1257400917, 'in_queue': 140.58814287185669}, 'user_level': 'admin', 'unique_id': 'stkendm7a.fnal.gov-1257400918-29514-9'} to mover
00:04:18 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  <storage groups {'ktev': [('9940B10.mover', 'VO6332'), ('9940B35.mover', 'VO6341')]} volume_families {'ktev.ktev_migration.cern': [('9940B10.mover', 'VO6332'), ('9940B35.mover', 'VO6341')]} >
00:04:18 stkenmvr35a.fnal.gov 005964 root I DBT35MV  READ FROM HSM Thread MainThread
00:04:18 stkendm7a.fnal.gov 029514 root I ENCP  Listening for control socket at: ('131.225.13.19', 50827) Thread READ
00:04:18 stkenmvr35a.fnal.gov 005964 root I DBT35MV  SENDING {'vc': {'comment': '', 'declared': 1083263105.0, 'blocksize': 131072, 'storage_group': 'ktev', 'library': 'CD-9940B', 'eod_cookie': '0000_000000000_0000172', 'current_location': '0000_000000000_0000029', 'user_inhibit': ['none', 'none'], 'sum_rd_access': 95, 'system_inhibit': ['none', 'migrating'], 'external_label': 'VO6341', 'wrapper': 'cern', 'remaining_bytes': 6809693696L, 'sum_mounts': 54, 'capacity_bytes': 214748364800L, 'media_type': '9940B', 'last_access': 1257400917.0, 'status': ('ok', None), 'si_time': [0.0, 1257399311.0], 'non_del_files': 57, 'sum_wr_err': 0, 'sum_wr_access': 57, 'file_family': 'ktev_migration', 'address': ('131.225.13.59', 7502), 'volume_family': 'ktev.ktev_migration.cern', 'modification_time': 1193243360.0, 'write_protected': 'y', 'sum_rd_err': 0, 'first_access': 1088544231.0}, 'outfilepath': '/data/data2/Migration_Spool/VO6341:0000_000000000_0000029', 'encp': {'agetime': 30, 'adminpri': 6, 'delpri': 1, 'basepri': 1000007, 'delayed_dismount': 2, 'curpri': 1000007}, 'file_size': 4235003240L, 'mover': {'max_buffer': 1048576000L, 'check_written_file': 10, 'max_failures': 5, 'library': 'CD-9940B.library_manager', 'local_mover': 0, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7835, 'mc_device': '0,0,10,1', 'compression': 0, 'serial_num': '479001001549', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 28311552.0, 'vendor_id': 'STK', 'driver': 'FTTDriver', 'host': 'stkenmvr35a', 'max_time_in_state': 1200, 'device': 'stkenmvr35a:/dev/rmt/tps2d0n', 'name': '9940B35.mover', 'send_stats': 1, 'product_id': 'T9940B', 'statistics_path': '/tmp/enstore/enstore/DBT35MV.stat', 'mover_address': ('131.225.13.87', 7835), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.87', 33898), 'data_ip': 'stkenmvr35a', 'logname': 'DBT35MV', 'hostip': '131.225.13.87', 'do_cleaning': 'No', 'media_changer': 'stk.media_changer'}, 'lm': {'address': ('131.225.13.4', 7522)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC084/f0', 'uid': 0, 'uname': 'root', 'rmajor': 0, 'machine': ('Linux', 'stkendm7a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (33188, 525142400L, 20L, 1, 5406, 2539, 1L, 1177385614, 1088549484, 1088547614), 'mode_octal': '0100744', 'mode': 33252, 'size_bytes': 4235003240L, 'gname': 'root', 'gid': 0, 'fullname': '/data/data2/Migration_Spool/VO6341:0000_000000000_0000029', 'inode': 525142400L, 'minor': 20}, 'version': 'v3_8  CVS $Revision: 1.963 $ encp.pyc', 'encp_daq': None, 'client_crc': 1, 'infile': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC084/.(access)(001F000000000000014D0980)', 'status': ('ok', None), 'volume': 'VO6341', 'outfile': '/data/data2/Migration_Spool/VO6341:0000_000000000_0000029', 'fc': {'status': ('ok', None), 'pnfsid': '001F000000000000014D0980', 'pnfs_name0': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC084/f0', 'deleted': 'no', 'external_label': 'VO6341', 'drive': 'stkenmvr26a:/dev/rmt/tps0d0n:4790000200', 'update': None, 'gid': 0, 'location_cookie': '0000_000000000_0000029', 'complete_crc': 1714586310L, 'address': ('131.225.13.59', 7501), 'uid': 0, 'bfid': 'CDMS108854948400000', 'sanity_cookie': (65536L, 3919951947L), 'size': 4235003240L}, 'infilepath': '/pnfs/fs/usr/ktev/migrated_from_dlt/QK/QKC0/QKC084/f0', 'at_the_top': 6, 'bfid': 'CDMS108854948400000', 'local_inode': 4472843L, 'resend': {'max_retry': 3, 'resubmits': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.19', 50827), 'work': 'read_from_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257401058.684545, 'encp_start_time': 1257400917.93977, 'job_queued': 1257400918.0959611, 't0': 1257400917, 'in_queue': 140.58814287185669}, 'user_level': 'admin', 'unique_id': 'stkendm7a.fnal.gov-1257400918-29514-9'} Thread client_connect_thread
00:04:18 stkendm7a.fnal.gov 029514 root I ENCP  Waiting for mover to connect control socket at address: ('131.225.13.19', 50827)  elapsed=1746.470sec Thread READ
00:04:18 stkendm7a.fnal.gov 029514 root I ENCP  Control socket ('131.225.13.19', 50827) is connected to ('131.225.13.87', 33899) for stkendm7a.fnal.gov-1257400918-29514-9.  elapsed=1746.480sec Thread READ
00:04:18 stkendm7a.fnal.gov 029514 root I ENCP  Received callback ticket from mover 9940B35.mover for transfer stkendm7a.fnal.gov-1257400918-29514-9.  elapsed=1746.490sec Thread READ
00:04:18 stkendm7a.fnal.gov 029514 root I ENCP  Attempting to connect data socket to mover at ('131.225.13.87', 33898). Thread READ
00:04:18 stkendm7a.fnal.gov 029514 root I ENCP  Connecting data socket to mover (('131.225.13.87', 33898)) with interface 131.225.13.19.  elapsed=1746.490sec Thread READ
00:04:18 stkendm7a.fnal.gov 029514 root I ENCP  Data socket ('131.225.13.19', 50836) is connected to ('131.225.13.87', 33898).  elapsed=1746.490sec Thread READ
00:04:18 stkendm7a.fnal.gov 029514 root I ENCP  Opening local file. Thread READ
00:04:18 stkenmvr35a.fnal.gov 005964 root I DBT35MV  tape position: target 29 current 26 1 Thread seek_thread
00:04:18 stkenmvr35a.fnal.gov 005964 root I DBT35MV  skip_fm 3 Thread seek_thread
00:04:18 stkenmvr35a.fnal.gov 005964 root I DBT35MV  read_tape starting, bytes_to_read=4235003240 Thread tape_thread
00:04:19 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:04:19 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  <storage groups {'ktev': [('9940B10.mover', 'VO6332'), ('9940B35.mover', 'VO6341')]} volume_families {'ktev.ktev_migration.cern': [('9940B10.mover', 'VO6332'), ('9940B35.mover', 'VO6341')]} >
00:04:19 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:04:21 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:04:21 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:04:21 stkendca22a.fnal.gov 021022 root I ENCP  Version: v3_8b  CVS $Revision: 1.922 $ <frozen>  OS: Linux 2.6.9-89.0.9.ELsmp i686 Release:  Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0)  Group: root(0)  Euser: root(0)  Egroup: root(0)  Library: CD-LTO4G1  Storage Group: minos  File Family: minos  FF Wrapper: cpio_odc  FF Width: 1  Current working directory: stkendca22a.fnal.gov:/home/enstore/dcache-log  Command line: /fnal/ups/prd/encp/v3_8b/Linux-2-6-dcache/encp --verbose 4 --threaded --ecrc --bypass-filesystem-max-filesize-check --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/713/n13047134_0009_L010185R_D07_r4.cand.dogwood1.0.root --put-cache 000F0000000000000B2472F8 /diskb/write-pool-2/data/000F0000000000000B2472F8
00:04:22 stkendca22a.fnal.gov 021258 root I ENCP  Version: v3_8b  CVS $Revision: 1.922 $ <frozen>  OS: Linux 2.6.9-89.0.9.ELsmp i686 Release:  Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0)  Group: root(0)  Euser: root(0)  Egroup: root(0)  Library: garbage  Storage Group: test  File Family: dcache  FF Wrapper: cpio_odc  FF Width: 1  Current working directory: stkendca22a.fnal.gov:/home/enstore/dcache-log  Command line: /fnal/ups/prd/encp/v3_8b/Linux-2-6-dcache/encp --verbose 4 --threaded --ecrc --bypass-filesystem-max-filesize-check --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0002 --put-cache 0005000000000000005FA5C8 /diskb/write-pool-2/data/0005000000000000005FA5C8
00:04:22 stkendca22a.fnal.gov 021281 root I ENCP  Version: v3_8b  CVS $Revision: 1.922 $ <frozen>  OS: Linux 2.6.9-89.0.9.ELsmp i686 Release:  Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0)  Group: root(0)  Euser: root(0)  Egroup: root(0)  Library: garbage  Storage Group: test  File Family: dcache  FF Wrapper: cpio_odc  FF Width: 1  Current working directory: stkendca22a.fnal.gov:/home/enstore/dcache-log  Command line: /fnal/ups/prd/encp/v3_8b/Linux-2-6-dcache/encp --verbose 4 --threaded --ecrc --bypass-filesystem-max-filesize-check --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0003 --put-cache 0005000000000000005FA5F8 /diskb/write-pool-2/data/0005000000000000005FA5F8
00:04:22 stkendca22a.fnal.gov 021323 root I ENCP  Version: v3_8b  CVS $Revision: 1.922 $ <frozen>  OS: Linux 2.6.9-89.0.9.ELsmp i686 Release:  Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0)  Group: root(0)  Euser: root(0)  Egroup: root(0)  Library: garbage  Storage Group: test  File Family: dcache  FF Wrapper: cpio_odc  FF Width: 1  Current working directory: stkendca22a.fnal.gov:/home/enstore/dcache-log  Command line: /fnal/ups/prd/encp/v3_8b/Linux-2-6-dcache/encp --verbose 4 --threaded --ecrc --bypass-filesystem-max-filesize-check --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0001 --put-cache 0005000000000000005FA590 /diskb/write-pool-2/data/0005000000000000005FA590
00:04:22 stkendca22a.fnal.gov 021022 root I ENCP  Creating zero length output files.
00:04:22 stkendca22a.fnal.gov 021022 root I ENCP  Sending /pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/713/n13047134_0009_L010185R_D07_r4.cand.dogwood1.0.root write request to LM: unique_id: stkendca22a.fnal.gov-1257401062-21022-0 inputfile: /diskb/write-pool-2/data/000F0000000000000B2472F8 outputfile: /pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/713/.(access)(000F0000000000000B2472F8)
00:04:22 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  MSG_TYPE=ADD_TO_LMQ  request added to pending queue for storage group : minos
00:04:22 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  write Q'd /diskb/write-pool-2/data/000F0000000000000B2472F8 (stkendca22a.fnal.gov-1257401062-21022-0) -> /pnfs/fnal.gov/usr/minos/mcout_data/dogwood1/near/daikon_07/L010185R_r4/cand_data/713/n13047134_0009_L010185R_D07_r4.cand.dogwood1.0.root : library=CD-LTO4G1 family=minos requester:root volume_family:minos.minos.cpio_odc
00:04:22 stkendca22a.fnal.gov 021022 root I ENCP  Listening for control socket at: ('131.225.13.100', 40234)
00:04:22 stkendca22a.fnal.gov 021022 root I ENCP  Waiting for mover to connect control socket at address: ('131.225.13.100', 40234)  elapsed=0.820sec
00:04:22 stkendca22a.fnal.gov 021258 root I ENCP  Configuration Server: no such name: 'garbage.library_manager'
00:04:22 stkendca22a.fnal.gov 021258 root E ENCP  INFILE=/diskb/write-pool-2/data/0005000000000000005FA5C8 OUTFILE=/pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0002 FILESIZE=801512 LABEL= LOCATION= DRIVE= DRIVE_SN= TRANSFER_TIME=0.00 SEEK_TIME=0.00 MOUNT_TIME=0.00 QWAIT_TIME=0.00 TIME2NOW=0.94 CRC= STATUS=KEYERROR Configuration Server: no such name: 'garbage.library_manager'
00:04:22 stkendca22a.fnal.gov 021281 root I ENCP  Configuration Server: no such name: 'garbage.library_manager'
00:04:22 stkendca22a.fnal.gov 021281 root E ENCP  INFILE=/diskb/write-pool-2/data/0005000000000000005FA5F8 OUTFILE=/pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0003 FILESIZE=801512 LABEL= LOCATION= DRIVE= DRIVE_SN= TRANSFER_TIME=0.00 SEEK_TIME=0.00 MOUNT_TIME=0.00 QWAIT_TIME=0.00 TIME2NOW=0.96 CRC= STATUS=KEYERROR Configuration Server: no such name: 'garbage.library_manager'
00:04:22 stkendca22a.fnal.gov 021323 root I ENCP  Configuration Server: no such name: 'garbage.library_manager'
00:04:22 stkendca22a.fnal.gov 021323 root E ENCP  INFILE=/diskb/write-pool-2/data/0005000000000000005FA590 OUTFILE=/pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0001 FILESIZE=801512 LABEL= LOCATION= DRIVE= DRIVE_SN= TRANSFER_TIME=0.00 SEEK_TIME=0.00 MOUNT_TIME=0.00 QWAIT_TIME=0.00 TIME2NOW=0.99 CRC= STATUS=KEYERROR Configuration Server: no such name: 'garbage.library_manager'
00:04:23 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:04:23 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:04:25 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:04:25 stkensrv4n.fnal.gov 005795 enstore I 9940BLM  <storage groups {'ktev': [('9940B35.mover', 'VO6341'), ('9940B10.mover', 'VO6332')]} volume_families {'ktev.ktev_migration.cern': [('9940B35.mover', 'VO6341'), ('9940B10.mover', 'VO6332')]} >
00:04:25 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:04:27 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:04:27 stkenmvr217a.fnal.gov 007075 root I LTO4_117MV  tape position: target 5289 current 0 1 Thread seek_thread
00:04:27 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:04:29 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:04:29 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  Tape is ejected Thread media_thread
00:04:29 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  dismounting VON262 Thread media_thread
00:04:29 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC  PROCESSING dismount VON262 0,5,1,7
00:04:29 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:04:31 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:04:31 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:04:32 stkensrv4n.fnal.gov 007073 enstore I SL8500G1MC  query drive 0,5,1,7 => 0,0,5,1,7 online          in use      VON262     IBM-LTO4
00:04:33 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:04:33 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:04:33 stkensrv4n.fnal.gov 005357 enstore E LTO4G1LM  MOVER ERROR RQ {'status': ('ok', None), 'returned_work': None, 'library': None, 'mover_type': '', 'current_location': '0000_000000000_0000000', 'ip_map': '', 'operation': None, 'read_only': 0, 'current_time': 1257401073.877764, 'error_source': None, 'external_label': None, 'state': 'OFFLINE', 'r_a': (('131.225.13.194', 54880), 16283L, '131.225.13.194-54880-1256586929.509013-4866'), 'library_list': [('CD-LTO4G1.library_manager', ('131.225.13.4', 8541))], 'current_priority': (None, None), 'transfer_deficiency': 1, 'time_in_state': 814143.51770591736, 'address': ('131.225.13.194', 7860), 'volume_family': None, 'work': 'mover_error', 'mover': 'LTO4_99.mover', 'volume_status': (['none', 'none'], ['none', 'none']), 'unique_id': None}
00:04:35 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:04:35 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:04:37 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:04:37 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:04:39 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:04:39 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:04:40 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_115.mover', 'VOO437'), ('LTO4_117.mover', 'VOO096')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:04:40 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:04:41 last message repeated 1 times
00:04:41 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:04:43 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:04:43 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:04:45 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:04:45 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:04:47 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:04:47 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:04:49 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  <storage groups {'exp-db': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'cms': [('LTO4_46.mover', 'VOP775')], 'minos': [('LTO4_109.mover', 'VON262')]} volume_families {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_105.mover', 'VON266'), ('LTO4_42.mover', 'VON743')], 'minos.minos.cpio_odc': [('LTO4_109.mover', 'VON262')], 'cms.cms7.cpio_odc': [('LTO4_46.mover', 'VOP775')]} >
00:04:49 stkendca18a.fnal.gov 022164 root I ENCP  File /diskb/write-pool-2/data/0030000000000000061EB018 transfered for stkendca18a.fnal.gov-1257400927-22164-0 in 81.6372010708 seconds.
00:04:49 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM  <storage groups {'miniboone': [('LTO4_117.mover', 'VOO096'), ('LTO4_115.mover', 'VOO437')]} volume_families {'miniboone.TankData.cpio_odc': [('LTO4_117.mover', 'VOO096')], 'miniboone.OpenRootTree.cpio_odc': [('LTO4_115.mover', 'VOO437')]} >
00:04:50 stkensrv4n.fnal.gov 007073 enstore I SL8500G1MC  dismount VOLUME 0,5,1,7 force => 0,Dismount: Forced dismount of VON262 from   0, 5, 1, 7
00:04:50 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC  FINISHED dismount VON262 0,5,1,7 returned ('ok', 0, 'dismount VOLUME 0,5,1,7 force => 0,Dismount: Forced dismount of VON262 from   0, 5, 1, 7', '', '')
00:04:50 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV  dismounted VON262 ULTRIUM-TD4 00:04:50 Thread media_thread
00:04:50 stkensrv0n.fnal.gov 028762 enstore I VOLSRV  Assigning  volume VON262 from storage group minos.minos.cpio_odc to library CD-LTO4G1, volume family minos.minos.cpio_odc
00:04:50 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  write_to_hsm work on vol=VON262 mover=LTO4_109.mover requester:root
00:04:50 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM  MSG_TYPE=ADD_TO_LMQ  request added to work at movers queue for storage group : minos
00:04:50 stkensrv4n.fnal.gov 005357 enstore I LTO