ocfs2_dlmfs: Use the stackglue.
[pandora-kernel.git] / fs / ocfs2 / dlmfs / userdlm.c
1 /* -*- mode: c; c-basic-offset: 8; -*-
2  * vim: noexpandtab sw=8 ts=8 sts=0:
3  *
4  * userdlm.c
5  *
6  * Code which implements the kernel side of a minimal userspace
7  * interface to our DLM.
8  *
9  * Many of the functions here are pared down versions of dlmglue.c
10  * functions.
11  *
12  * Copyright (C) 2003, 2004 Oracle.  All rights reserved.
13  *
14  * This program is free software; you can redistribute it and/or
15  * modify it under the terms of the GNU General Public
16  * License as published by the Free Software Foundation; either
17  * version 2 of the License, or (at your option) any later version.
18  *
19  * This program is distributed in the hope that it will be useful,
20  * but WITHOUT ANY WARRANTY; without even the implied warranty of
21  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
22  * General Public License for more details.
23  *
24  * You should have received a copy of the GNU General Public
25  * License along with this program; if not, write to the
26  * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
27  * Boston, MA 021110-1307, USA.
28  */
29
30 #include <linux/signal.h>
31
32 #include <linux/module.h>
33 #include <linux/fs.h>
34 #include <linux/types.h>
35 #include <linux/crc32.h>
36
37 #include "ocfs2_lockingver.h"
38 #include "stackglue.h"
39 #include "userdlm.h"
40
41 #define MLOG_MASK_PREFIX ML_DLMFS
42 #include "cluster/masklog.h"
43
44
45 static inline struct user_lock_res *user_lksb_to_lock_res(struct ocfs2_dlm_lksb *lksb)
46 {
47         return container_of(lksb, struct user_lock_res, l_lksb);
48 }
49
50 static inline int user_check_wait_flag(struct user_lock_res *lockres,
51                                        int flag)
52 {
53         int ret;
54
55         spin_lock(&lockres->l_lock);
56         ret = lockres->l_flags & flag;
57         spin_unlock(&lockres->l_lock);
58
59         return ret;
60 }
61
62 static inline void user_wait_on_busy_lock(struct user_lock_res *lockres)
63
64 {
65         wait_event(lockres->l_event,
66                    !user_check_wait_flag(lockres, USER_LOCK_BUSY));
67 }
68
69 static inline void user_wait_on_blocked_lock(struct user_lock_res *lockres)
70
71 {
72         wait_event(lockres->l_event,
73                    !user_check_wait_flag(lockres, USER_LOCK_BLOCKED));
74 }
75
76 /* I heart container_of... */
77 static inline struct ocfs2_cluster_connection *
78 cluster_connection_from_user_lockres(struct user_lock_res *lockres)
79 {
80         struct dlmfs_inode_private *ip;
81
82         ip = container_of(lockres,
83                           struct dlmfs_inode_private,
84                           ip_lockres);
85         return ip->ip_conn;
86 }
87
88 static struct inode *
89 user_dlm_inode_from_user_lockres(struct user_lock_res *lockres)
90 {
91         struct dlmfs_inode_private *ip;
92
93         ip = container_of(lockres,
94                           struct dlmfs_inode_private,
95                           ip_lockres);
96         return &ip->ip_vfs_inode;
97 }
98
99 static inline void user_recover_from_dlm_error(struct user_lock_res *lockres)
100 {
101         spin_lock(&lockres->l_lock);
102         lockres->l_flags &= ~USER_LOCK_BUSY;
103         spin_unlock(&lockres->l_lock);
104 }
105
106 #define user_log_dlm_error(_func, _stat, _lockres) do {                 \
107         mlog(ML_ERROR, "Dlm error %d while calling %s on "              \
108                 "resource %.*s\n", _stat, _func,                        \
109                 _lockres->l_namelen, _lockres->l_name);                 \
110 } while (0)
111
112 /* WARNING: This function lives in a world where the only three lock
113  * levels are EX, PR, and NL. It *will* have to be adjusted when more
114  * lock types are added. */
115 static inline int user_highest_compat_lock_level(int level)
116 {
117         int new_level = DLM_LOCK_EX;
118
119         if (level == DLM_LOCK_EX)
120                 new_level = DLM_LOCK_NL;
121         else if (level == DLM_LOCK_PR)
122                 new_level = DLM_LOCK_PR;
123         return new_level;
124 }
125
126 static void user_ast(struct ocfs2_dlm_lksb *lksb)
127 {
128         struct user_lock_res *lockres = user_lksb_to_lock_res(lksb);
129         int status;
130
131         mlog(0, "AST fired for lockres %.*s\n", lockres->l_namelen,
132              lockres->l_name);
133
134         spin_lock(&lockres->l_lock);
135
136         status = ocfs2_dlm_lock_status(&lockres->l_lksb);
137         if (status) {
138                 mlog(ML_ERROR, "lksb status value of %u on lockres %.*s\n",
139                      status, lockres->l_namelen, lockres->l_name);
140                 spin_unlock(&lockres->l_lock);
141                 return;
142         }
143
144         mlog_bug_on_msg(lockres->l_requested == DLM_LOCK_IV,
145                         "Lockres %.*s, requested ivmode. flags 0x%x\n",
146                         lockres->l_namelen, lockres->l_name, lockres->l_flags);
147
148         /* we're downconverting. */
149         if (lockres->l_requested < lockres->l_level) {
150                 if (lockres->l_requested <=
151                     user_highest_compat_lock_level(lockres->l_blocking)) {
152                         lockres->l_blocking = DLM_LOCK_NL;
153                         lockres->l_flags &= ~USER_LOCK_BLOCKED;
154                 }
155         }
156
157         lockres->l_level = lockres->l_requested;
158         lockres->l_requested = DLM_LOCK_IV;
159         lockres->l_flags |= USER_LOCK_ATTACHED;
160         lockres->l_flags &= ~USER_LOCK_BUSY;
161
162         spin_unlock(&lockres->l_lock);
163
164         wake_up(&lockres->l_event);
165 }
166
167 static inline void user_dlm_grab_inode_ref(struct user_lock_res *lockres)
168 {
169         struct inode *inode;
170         inode = user_dlm_inode_from_user_lockres(lockres);
171         if (!igrab(inode))
172                 BUG();
173 }
174
175 static void user_dlm_unblock_lock(struct work_struct *work);
176
177 static void __user_dlm_queue_lockres(struct user_lock_res *lockres)
178 {
179         if (!(lockres->l_flags & USER_LOCK_QUEUED)) {
180                 user_dlm_grab_inode_ref(lockres);
181
182                 INIT_WORK(&lockres->l_work, user_dlm_unblock_lock);
183
184                 queue_work(user_dlm_worker, &lockres->l_work);
185                 lockres->l_flags |= USER_LOCK_QUEUED;
186         }
187 }
188
189 static void __user_dlm_cond_queue_lockres(struct user_lock_res *lockres)
190 {
191         int queue = 0;
192
193         if (!(lockres->l_flags & USER_LOCK_BLOCKED))
194                 return;
195
196         switch (lockres->l_blocking) {
197         case DLM_LOCK_EX:
198                 if (!lockres->l_ex_holders && !lockres->l_ro_holders)
199                         queue = 1;
200                 break;
201         case DLM_LOCK_PR:
202                 if (!lockres->l_ex_holders)
203                         queue = 1;
204                 break;
205         default:
206                 BUG();
207         }
208
209         if (queue)
210                 __user_dlm_queue_lockres(lockres);
211 }
212
213 static void user_bast(struct ocfs2_dlm_lksb *lksb, int level)
214 {
215         struct user_lock_res *lockres = user_lksb_to_lock_res(lksb);
216
217         mlog(0, "Blocking AST fired for lockres %.*s. Blocking level %d\n",
218              lockres->l_namelen, lockres->l_name, level);
219
220         spin_lock(&lockres->l_lock);
221         lockres->l_flags |= USER_LOCK_BLOCKED;
222         if (level > lockres->l_blocking)
223                 lockres->l_blocking = level;
224
225         __user_dlm_queue_lockres(lockres);
226         spin_unlock(&lockres->l_lock);
227
228         wake_up(&lockres->l_event);
229 }
230
231 static void user_unlock_ast(struct ocfs2_dlm_lksb *lksb, int status)
232 {
233         struct user_lock_res *lockres = user_lksb_to_lock_res(lksb);
234
235         mlog(0, "UNLOCK AST called on lock %.*s\n", lockres->l_namelen,
236              lockres->l_name);
237
238         if (status)
239                 mlog(ML_ERROR, "dlm returns status %d\n", status);
240
241         spin_lock(&lockres->l_lock);
242         /* The teardown flag gets set early during the unlock process,
243          * so test the cancel flag to make sure that this ast isn't
244          * for a concurrent cancel. */
245         if (lockres->l_flags & USER_LOCK_IN_TEARDOWN
246             && !(lockres->l_flags & USER_LOCK_IN_CANCEL)) {
247                 lockres->l_level = DLM_LOCK_IV;
248         } else if (status == DLM_CANCELGRANT) {
249                 /* We tried to cancel a convert request, but it was
250                  * already granted. Don't clear the busy flag - the
251                  * ast should've done this already. */
252                 BUG_ON(!(lockres->l_flags & USER_LOCK_IN_CANCEL));
253                 lockres->l_flags &= ~USER_LOCK_IN_CANCEL;
254                 goto out_noclear;
255         } else {
256                 BUG_ON(!(lockres->l_flags & USER_LOCK_IN_CANCEL));
257                 /* Cancel succeeded, we want to re-queue */
258                 lockres->l_requested = DLM_LOCK_IV; /* cancel an
259                                                     * upconvert
260                                                     * request. */
261                 lockres->l_flags &= ~USER_LOCK_IN_CANCEL;
262                 /* we want the unblock thread to look at it again
263                  * now. */
264                 if (lockres->l_flags & USER_LOCK_BLOCKED)
265                         __user_dlm_queue_lockres(lockres);
266         }
267
268         lockres->l_flags &= ~USER_LOCK_BUSY;
269 out_noclear:
270         spin_unlock(&lockres->l_lock);
271
272         wake_up(&lockres->l_event);
273 }
274
275 /*
276  * This is the userdlmfs locking protocol version.
277  *
278  * See fs/ocfs2/dlmglue.c for more details on locking versions.
279  */
280 static struct ocfs2_locking_protocol user_dlm_lproto = {
281         .lp_max_version = {
282                 .pv_major = OCFS2_LOCKING_PROTOCOL_MAJOR,
283                 .pv_minor = OCFS2_LOCKING_PROTOCOL_MINOR,
284         },
285         .lp_lock_ast            = user_ast,
286         .lp_blocking_ast        = user_bast,
287         .lp_unlock_ast          = user_unlock_ast,
288 };
289
290 static inline void user_dlm_drop_inode_ref(struct user_lock_res *lockres)
291 {
292         struct inode *inode;
293         inode = user_dlm_inode_from_user_lockres(lockres);
294         iput(inode);
295 }
296
297 static void user_dlm_unblock_lock(struct work_struct *work)
298 {
299         int new_level, status;
300         struct user_lock_res *lockres =
301                 container_of(work, struct user_lock_res, l_work);
302         struct ocfs2_cluster_connection *conn =
303                 cluster_connection_from_user_lockres(lockres);
304
305         mlog(0, "processing lockres %.*s\n", lockres->l_namelen,
306              lockres->l_name);
307
308         spin_lock(&lockres->l_lock);
309
310         mlog_bug_on_msg(!(lockres->l_flags & USER_LOCK_QUEUED),
311                         "Lockres %.*s, flags 0x%x\n",
312                         lockres->l_namelen, lockres->l_name, lockres->l_flags);
313
314         /* notice that we don't clear USER_LOCK_BLOCKED here. If it's
315          * set, we want user_ast clear it. */
316         lockres->l_flags &= ~USER_LOCK_QUEUED;
317
318         /* It's valid to get here and no longer be blocked - if we get
319          * several basts in a row, we might be queued by the first
320          * one, the unblock thread might run and clear the queued
321          * flag, and finally we might get another bast which re-queues
322          * us before our ast for the downconvert is called. */
323         if (!(lockres->l_flags & USER_LOCK_BLOCKED)) {
324                 spin_unlock(&lockres->l_lock);
325                 goto drop_ref;
326         }
327
328         if (lockres->l_flags & USER_LOCK_IN_TEARDOWN) {
329                 spin_unlock(&lockres->l_lock);
330                 goto drop_ref;
331         }
332
333         if (lockres->l_flags & USER_LOCK_BUSY) {
334                 if (lockres->l_flags & USER_LOCK_IN_CANCEL) {
335                         spin_unlock(&lockres->l_lock);
336                         goto drop_ref;
337                 }
338
339                 lockres->l_flags |= USER_LOCK_IN_CANCEL;
340                 spin_unlock(&lockres->l_lock);
341
342                 status = ocfs2_dlm_unlock(conn, &lockres->l_lksb,
343                                           DLM_LKF_CANCEL);
344                 if (status)
345                         user_log_dlm_error("ocfs2_dlm_unlock", status, lockres);
346                 goto drop_ref;
347         }
348
349         /* If there are still incompat holders, we can exit safely
350          * without worrying about re-queueing this lock as that will
351          * happen on the last call to user_cluster_unlock. */
352         if ((lockres->l_blocking == DLM_LOCK_EX)
353             && (lockres->l_ex_holders || lockres->l_ro_holders)) {
354                 spin_unlock(&lockres->l_lock);
355                 mlog(0, "can't downconvert for ex: ro = %u, ex = %u\n",
356                         lockres->l_ro_holders, lockres->l_ex_holders);
357                 goto drop_ref;
358         }
359
360         if ((lockres->l_blocking == DLM_LOCK_PR)
361             && lockres->l_ex_holders) {
362                 spin_unlock(&lockres->l_lock);
363                 mlog(0, "can't downconvert for pr: ex = %u\n",
364                         lockres->l_ex_holders);
365                 goto drop_ref;
366         }
367
368         /* yay, we can downconvert now. */
369         new_level = user_highest_compat_lock_level(lockres->l_blocking);
370         lockres->l_requested = new_level;
371         lockres->l_flags |= USER_LOCK_BUSY;
372         mlog(0, "Downconvert lock from %d to %d\n",
373                 lockres->l_level, new_level);
374         spin_unlock(&lockres->l_lock);
375
376         /* need lock downconvert request now... */
377         status = ocfs2_dlm_lock(conn, new_level, &lockres->l_lksb,
378                                 DLM_LKF_CONVERT|DLM_LKF_VALBLK,
379                                 lockres->l_name,
380                                 lockres->l_namelen);
381         if (status) {
382                 user_log_dlm_error("ocfs2_dlm_lock", status, lockres);
383                 user_recover_from_dlm_error(lockres);
384         }
385
386 drop_ref:
387         user_dlm_drop_inode_ref(lockres);
388 }
389
390 static inline void user_dlm_inc_holders(struct user_lock_res *lockres,
391                                         int level)
392 {
393         switch(level) {
394         case DLM_LOCK_EX:
395                 lockres->l_ex_holders++;
396                 break;
397         case DLM_LOCK_PR:
398                 lockres->l_ro_holders++;
399                 break;
400         default:
401                 BUG();
402         }
403 }
404
405 /* predict what lock level we'll be dropping down to on behalf
406  * of another node, and return true if the currently wanted
407  * level will be compatible with it. */
408 static inline int
409 user_may_continue_on_blocked_lock(struct user_lock_res *lockres,
410                                   int wanted)
411 {
412         BUG_ON(!(lockres->l_flags & USER_LOCK_BLOCKED));
413
414         return wanted <= user_highest_compat_lock_level(lockres->l_blocking);
415 }
416
417 int user_dlm_cluster_lock(struct user_lock_res *lockres,
418                           int level,
419                           int lkm_flags)
420 {
421         int status, local_flags;
422         struct ocfs2_cluster_connection *conn =
423                 cluster_connection_from_user_lockres(lockres);
424
425         if (level != DLM_LOCK_EX &&
426             level != DLM_LOCK_PR) {
427                 mlog(ML_ERROR, "lockres %.*s: invalid request!\n",
428                      lockres->l_namelen, lockres->l_name);
429                 status = -EINVAL;
430                 goto bail;
431         }
432
433         mlog(0, "lockres %.*s: asking for %s lock, passed flags = 0x%x\n",
434              lockres->l_namelen, lockres->l_name,
435              (level == DLM_LOCK_EX) ? "DLM_LOCK_EX" : "DLM_LOCK_PR",
436              lkm_flags);
437
438 again:
439         if (signal_pending(current)) {
440                 status = -ERESTARTSYS;
441                 goto bail;
442         }
443
444         spin_lock(&lockres->l_lock);
445
446         /* We only compare against the currently granted level
447          * here. If the lock is blocked waiting on a downconvert,
448          * we'll get caught below. */
449         if ((lockres->l_flags & USER_LOCK_BUSY) &&
450             (level > lockres->l_level)) {
451                 /* is someone sitting in dlm_lock? If so, wait on
452                  * them. */
453                 spin_unlock(&lockres->l_lock);
454
455                 user_wait_on_busy_lock(lockres);
456                 goto again;
457         }
458
459         if ((lockres->l_flags & USER_LOCK_BLOCKED) &&
460             (!user_may_continue_on_blocked_lock(lockres, level))) {
461                 /* is the lock is currently blocked on behalf of
462                  * another node */
463                 spin_unlock(&lockres->l_lock);
464
465                 user_wait_on_blocked_lock(lockres);
466                 goto again;
467         }
468
469         if (level > lockres->l_level) {
470                 local_flags = lkm_flags | DLM_LKF_VALBLK;
471                 if (lockres->l_level != DLM_LOCK_IV)
472                         local_flags |= DLM_LKF_CONVERT;
473
474                 lockres->l_requested = level;
475                 lockres->l_flags |= USER_LOCK_BUSY;
476                 spin_unlock(&lockres->l_lock);
477
478                 BUG_ON(level == DLM_LOCK_IV);
479                 BUG_ON(level == DLM_LOCK_NL);
480
481                 /* call dlm_lock to upgrade lock now */
482                 status = ocfs2_dlm_lock(conn, level, &lockres->l_lksb,
483                                         local_flags, lockres->l_name,
484                                         lockres->l_namelen);
485                 if (status) {
486                         if ((lkm_flags & DLM_LKF_NOQUEUE) &&
487                             (status != -EAGAIN))
488                                 user_log_dlm_error("ocfs2_dlm_lock",
489                                                    status, lockres);
490                         user_recover_from_dlm_error(lockres);
491                         goto bail;
492                 }
493
494                 user_wait_on_busy_lock(lockres);
495                 goto again;
496         }
497
498         user_dlm_inc_holders(lockres, level);
499         spin_unlock(&lockres->l_lock);
500
501         status = 0;
502 bail:
503         return status;
504 }
505
506 static inline void user_dlm_dec_holders(struct user_lock_res *lockres,
507                                         int level)
508 {
509         switch(level) {
510         case DLM_LOCK_EX:
511                 BUG_ON(!lockres->l_ex_holders);
512                 lockres->l_ex_holders--;
513                 break;
514         case DLM_LOCK_PR:
515                 BUG_ON(!lockres->l_ro_holders);
516                 lockres->l_ro_holders--;
517                 break;
518         default:
519                 BUG();
520         }
521 }
522
523 void user_dlm_cluster_unlock(struct user_lock_res *lockres,
524                              int level)
525 {
526         if (level != DLM_LOCK_EX &&
527             level != DLM_LOCK_PR) {
528                 mlog(ML_ERROR, "lockres %.*s: invalid request!\n",
529                      lockres->l_namelen, lockres->l_name);
530                 return;
531         }
532
533         spin_lock(&lockres->l_lock);
534         user_dlm_dec_holders(lockres, level);
535         __user_dlm_cond_queue_lockres(lockres);
536         spin_unlock(&lockres->l_lock);
537 }
538
539 void user_dlm_write_lvb(struct inode *inode,
540                         const char *val,
541                         unsigned int len)
542 {
543         struct user_lock_res *lockres = &DLMFS_I(inode)->ip_lockres;
544         char *lvb;
545
546         BUG_ON(len > DLM_LVB_LEN);
547
548         spin_lock(&lockres->l_lock);
549
550         BUG_ON(lockres->l_level < DLM_LOCK_EX);
551         lvb = ocfs2_dlm_lvb(&lockres->l_lksb);
552         memcpy(lvb, val, len);
553
554         spin_unlock(&lockres->l_lock);
555 }
556
557 ssize_t user_dlm_read_lvb(struct inode *inode,
558                           char *val,
559                           unsigned int len)
560 {
561         struct user_lock_res *lockres = &DLMFS_I(inode)->ip_lockres;
562         char *lvb;
563         ssize_t ret = len;
564
565         BUG_ON(len > DLM_LVB_LEN);
566
567         spin_lock(&lockres->l_lock);
568
569         BUG_ON(lockres->l_level < DLM_LOCK_PR);
570         if (ocfs2_dlm_lvb_valid(&lockres->l_lksb)) {
571                 lvb = ocfs2_dlm_lvb(&lockres->l_lksb);
572                 memcpy(val, lvb, len);
573         } else
574                 ret = 0;
575
576         spin_unlock(&lockres->l_lock);
577         return ret;
578 }
579
580 void user_dlm_lock_res_init(struct user_lock_res *lockres,
581                             struct dentry *dentry)
582 {
583         memset(lockres, 0, sizeof(*lockres));
584
585         spin_lock_init(&lockres->l_lock);
586         init_waitqueue_head(&lockres->l_event);
587         lockres->l_level = DLM_LOCK_IV;
588         lockres->l_requested = DLM_LOCK_IV;
589         lockres->l_blocking = DLM_LOCK_IV;
590
591         /* should have been checked before getting here. */
592         BUG_ON(dentry->d_name.len >= USER_DLM_LOCK_ID_MAX_LEN);
593
594         memcpy(lockres->l_name,
595                dentry->d_name.name,
596                dentry->d_name.len);
597         lockres->l_namelen = dentry->d_name.len;
598 }
599
600 int user_dlm_destroy_lock(struct user_lock_res *lockres)
601 {
602         int status = -EBUSY;
603         struct ocfs2_cluster_connection *conn =
604                 cluster_connection_from_user_lockres(lockres);
605
606         mlog(0, "asked to destroy %.*s\n", lockres->l_namelen, lockres->l_name);
607
608         spin_lock(&lockres->l_lock);
609         if (lockres->l_flags & USER_LOCK_IN_TEARDOWN) {
610                 spin_unlock(&lockres->l_lock);
611                 return 0;
612         }
613
614         lockres->l_flags |= USER_LOCK_IN_TEARDOWN;
615
616         while (lockres->l_flags & USER_LOCK_BUSY) {
617                 spin_unlock(&lockres->l_lock);
618
619                 user_wait_on_busy_lock(lockres);
620
621                 spin_lock(&lockres->l_lock);
622         }
623
624         if (lockres->l_ro_holders || lockres->l_ex_holders) {
625                 spin_unlock(&lockres->l_lock);
626                 goto bail;
627         }
628
629         status = 0;
630         if (!(lockres->l_flags & USER_LOCK_ATTACHED)) {
631                 spin_unlock(&lockres->l_lock);
632                 goto bail;
633         }
634
635         lockres->l_flags &= ~USER_LOCK_ATTACHED;
636         lockres->l_flags |= USER_LOCK_BUSY;
637         spin_unlock(&lockres->l_lock);
638
639         status = ocfs2_dlm_unlock(conn, &lockres->l_lksb, DLM_LKF_VALBLK);
640         if (status) {
641                 user_log_dlm_error("ocfs2_dlm_unlock", status, lockres);
642                 goto bail;
643         }
644
645         user_wait_on_busy_lock(lockres);
646
647         status = 0;
648 bail:
649         return status;
650 }
651
652 static void user_dlm_recovery_handler_noop(int node_num,
653                                            void *recovery_data)
654 {
655         /* We ignore recovery events */
656         return;
657 }
658
659 void user_dlm_set_locking_protocol(void)
660 {
661         ocfs2_stack_glue_set_max_proto_version(&user_dlm_lproto.lp_max_version);
662 }
663
664 struct ocfs2_cluster_connection *user_dlm_register(struct qstr *name)
665 {
666         int rc;
667         struct ocfs2_cluster_connection *conn;
668
669         rc = ocfs2_cluster_connect("o2cb", name->name, name->len,
670                                    &user_dlm_lproto,
671                                    user_dlm_recovery_handler_noop,
672                                    NULL, &conn);
673         if (rc)
674                 mlog_errno(rc);
675
676         return rc ? ERR_PTR(rc) : conn;
677 }
678
679 void user_dlm_unregister(struct ocfs2_cluster_connection *conn)
680 {
681         ocfs2_cluster_disconnect(conn, 0);
682 }